Index: llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -0,0 +1,919 @@ +//===------------ JITLink.h - JIT linker functionality ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains generic JIT-linker types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/MemoryBuffer.h" + +#include +#include +#include + +namespace llvm { +namespace jitlink { + +/// Base class for errors originating in JIT linker, e.g. missing relocation +/// support. +class JITLinkError : public ErrorInfo { +public: + static char ID; + + JITLinkError(Twine ErrMsg) : ErrMsg(ErrMsg.str()) {} + + void log(raw_ostream &OS) const override; + const std::string &getErrorMessage() const { return ErrMsg; } + std::error_code convertToErrorCode() const override; + +private: + std::string ErrMsg; +}; + +/// Manages allocations of JIT memory. +/// +/// Instances of this class may be accessed concurrently from multiple threads +/// and their implemetations should include any necessary synchronization. +class JITLinkMemoryManager { +public: + using ProtectionFlags = sys::Memory::ProtectionFlags; + + class SegmentRequest { + public: + SegmentRequest() = default; + SegmentRequest(size_t ContentSize, unsigned ContentAlign, + uint64_t ZeroFillSize, unsigned ZeroFillAlign) + : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize), + ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {} + size_t getContentSize() const { return ContentSize; } + unsigned getContentAlignment() const { return ContentAlign; } + uint64_t getZeroFillSize() const { return ZeroFillSize; } + unsigned getZeroFillAlignment() const { return ZeroFillAlign; } + + private: + size_t ContentSize = 0; + uint64_t ZeroFillSize = 0; + unsigned ContentAlign = 0; + unsigned ZeroFillAlign = 0; + }; + + using SegmentsRequestMap = DenseMap; + + using FinalizeContinuation = std::function; + + /// Represents an allocation created by the memory manager. + /// + /// An allocation object is responsible for allocating and owning jit-linker + /// working and target memory, and for transfering from working to target + /// memory. + /// + class Allocation { + public: + virtual ~Allocation(); + + /// Should return the address of linker working memory for the segment with + /// the given protection flags. + virtual MutableArrayRef getWorkingMemory(ProtectionFlags Seg) = 0; + + /// Should return the final address in the target process where the segment + /// will reside. + virtual JITTargetAddress getTargetMemory(ProtectionFlags Seg) = 0; + + /// Should transfer from working memory to target memory, and release + /// working memory. + virtual void finalizeAsync(FinalizeContinuation OnFinalize) = 0; + + /// Should deallocate target memory. + virtual Error deallocate() = 0; + }; + + virtual ~JITLinkMemoryManager(); + + /// Create an Allocation object. + virtual Expected> + allocate(const SegmentsRequestMap &Request) = 0; +}; + +// Forward declare the Atom class. +class Atom; + +/// Edge class. Represents both object file relocations, as well as layout and +/// keep-alive constraints. +class Edge { +public: + using Kind = uint8_t; + + using GenericEdgeKind = enum : Kind { + Invalid, // Invalid edge value. + FirstKeepAlive, // Keeps target alive. Offset/addend zero. + KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. + LayoutNext, // Layout constraint. Offset/Addend zero. + FirstRelocation // First architecture specific relocation. + }; + + using OffsetT = uint32_t; + using AddendT = int64_t; + + Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} + + OffsetT getOffset() const { return Offset; } + Kind getKind() const { return K; } + void setKind(Kind K) { this->K = K; } + bool isRelocation() const { return K >= FirstRelocation; } + Kind getRelocation() const { + assert(isRelocation() && "Not a relocation edge"); + return K - FirstRelocation; + } + bool isKeepAlive() const { return K >= FirstKeepAlive; } + Atom &getTarget() const { return *Target; } + void setTarget(Atom &Target) { this->Target = &Target; } + AddendT getAddend() const { return Addend; } + void setAddend(AddendT Addend) { this->Addend = Addend; } + +private: + Atom *Target; + OffsetT Offset; + AddendT Addend; + Kind K = 0; +}; + +using EdgeVector = std::vector; + +const StringRef getGenericEdgeKindName(Edge::Kind K); + +/// Base Atom class. Used by absolute and undefined atoms. +class Atom { + friend class AtomGraph; + +protected: + /// Create a named (as yet unresolved) atom. + Atom(StringRef Name) + : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false), + IsGlobal(false), IsAbsolute(false), IsCallable(false), + IsExported(false), IsWeak(false), HasLayoutNext(false), + IsCommon(false) {} + + /// Create an absolute symbol atom. + Atom(StringRef Name, JITTargetAddress Address) + : Name(Name), Address(Address), IsDefined(true), IsLive(false), + ShouldDiscard(false), IsGlobal(false), IsAbsolute(false), + IsCallable(false), IsExported(false), IsWeak(false), + HasLayoutNext(false), IsCommon(false) {} + +public: + /// Returns true if this atom has a name. + bool hasName() const { return Name != StringRef(); } + + /// Returns the name of this atom. + StringRef getName() const { return Name; } + + /// Returns the current target address of this atom. + /// The initial target address (for atoms that have one) will be taken from + /// the input object file's virtual address space. During the layout phase + /// of JIT linking the atom's address will be updated to point to its final + /// address in the JIT'd process. + JITTargetAddress getAddress() const { return Address; } + + /// Set the current target address of this atom. + void setAddress(JITTargetAddress Address) { this->Address = Address; } + + /// Returns true if this is a defined atom. + bool isDefined() const { return IsDefined; } + + /// Returns true if this atom is marked as live. + bool isLive() const { return IsLive; } + + /// Mark this atom as live. + /// + /// Note: Only defined and absolute atoms can be marked live. + void setLive(bool IsLive) { + assert((IsDefined || IsAbsolute || !IsLive) && + "Only defined and absolute atoms can be marked live"); + this->IsLive = IsLive; + } + + /// Returns true if this atom should be discarded during pruning. + bool shouldDiscard() const { return ShouldDiscard; } + + /// Mark this atom to be discarded. + /// + /// Note: Only defined and absolute atoms can be marked live. + void setShouldDiscard(bool ShouldDiscard) { + assert((IsDefined || IsAbsolute || !ShouldDiscard) && + "Only defined and absolute atoms can be marked live"); + this->ShouldDiscard = ShouldDiscard; + } + + /// Returns true if this definition is global (i.e. visible outside this + /// linkage unit). + /// + /// Note: This is distict from Exported, which means visibile outside the + /// JITDylib that this graph is being linked in to. + bool isGlobal() const { return IsGlobal; } + + /// Mark this atom as global. + void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; } + + /// Returns true if this atom represents an absolute symbol. + bool isAbsolute() const { return IsAbsolute; } + + /// Returns true if this atom is known to be callable. + /// + /// Primarily provided for easy interoperability with ORC, which uses the + /// JITSymbolFlags::Common flag to identify symbols that can be interposed + /// with stubs. + bool isCallable() const { return IsCallable; } + + /// Mark this atom as callable. + void setCallable(bool IsCallable) { + assert((IsDefined || IsAbsolute || !IsCallable) && + "Callable atoms must be defined or absolute"); + this->IsCallable = IsCallable; + } + + /// Returns true if this atom should appear in the symbol table of a final + /// linked image. + bool isExported() const { return IsExported; } + + /// Mark this atom as exported. + void setExported(bool IsExported) { + assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) && + "Exported atoms must have names"); + this->IsExported = IsExported; + } + + /// Returns true if this is a weak symbol. + bool isWeak() const { return IsWeak; } + + /// Mark this atom as weak. + void setWeak(bool IsWeak) { this->IsWeak = IsWeak; } + +private: + StringRef Name; + JITTargetAddress Address = 0; + + bool IsDefined : 1; + bool IsLive : 1; + bool ShouldDiscard : 1; + + bool IsGlobal : 1; + bool IsAbsolute : 1; + bool IsCallable : 1; + bool IsExported : 1; + bool IsWeak : 1; + +protected: + // These flags only make sense for DefinedAtom, but we can minimize the size + // of DefinedAtom by defining them here. + bool HasLayoutNext : 1; + bool IsCommon : 1; +}; + +// Forward declare DefinedAtom. +class DefinedAtom; + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A); +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName); + +/// Represents an object file section. +class Section { + friend class AtomGraph; + +private: + Section(StringRef Name, sys::Memory::ProtectionFlags Prot, unsigned Ordinal, + bool IsZeroFill) + : Name(Name), Prot(Prot), Ordinal(Ordinal), IsZeroFill(IsZeroFill) {} + + using DefinedAtomSet = DenseSet; + +public: + using atom_iterator = DefinedAtomSet::iterator; + using const_atom_iterator = DefinedAtomSet::const_iterator; + + ~Section(); + StringRef getName() const { return Name; } + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } + unsigned getSectionOrdinal() const { return Ordinal; } + size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } + + bool isZeroFill() const { return IsZeroFill; } + + /// Returns an iterator over the atoms in the section (in no particular + /// order). + iterator_range atoms() { + return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + } + + /// Returns an iterator over the atoms in the section (in no particular + /// order). + iterator_range atoms() const { + return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + } + + /// Return the number of atoms in this section. + DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); } + + /// Return true if this section contains no atoms. + bool atoms_empty() const { return DefinedAtoms.empty(); } + +private: + void addAtom(DefinedAtom &DA) { + assert(!DefinedAtoms.count(&DA) && "Atom is already in this section"); + DefinedAtoms.insert(&DA); + } + + void removeAtom(DefinedAtom &DA) { + assert(DefinedAtoms.count(&DA) && "Atom is not in this section"); + DefinedAtoms.erase(&DA); + } + + StringRef Name; + sys::Memory::ProtectionFlags Prot; + unsigned Ordinal = 0; + unsigned NextAtomOrdinal = 0; + bool IsZeroFill = false; + DefinedAtomSet DefinedAtoms; +}; + +/// Defined atom class. Suitable for use by defined named and anonymous +/// atoms. +class DefinedAtom : public Atom { + friend class AtomGraph; + +private: + DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) + : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), + Alignment(Alignment) {} + + DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, + uint32_t Alignment) + : Atom(Name, Address), Parent(Parent), + Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) {} + +public: + using edge_iterator = EdgeVector::iterator; + + Section &getSection() const { return Parent; } + + uint64_t getSize() const { return Size; } + + StringRef getContent() const { + assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom"); + assert(Size <= std::numeric_limits::max() && + "Content size too large"); + return {ContentPtr, Size}; + } + void setContent(StringRef Content) { + assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?"); + ContentPtr = Content.data(); + Size = Content.size(); + } + + bool isZeroFill() const { return Parent.isZeroFill(); } + + void setZeroFill(uint64_t Size) { + assert(Parent.isZeroFill() && !ContentPtr && + "Can't set zero-fill length of a non zero-fill atom"); + this->Size = Size; + } + + uint64_t getZeroFillSize() const { + assert(Parent.isZeroFill() && + "Can't get zero-fill length of a non zero-fill atom"); + return Size; + } + + uint32_t getAlignment() const { return Alignment; } + + bool hasLayoutNext() const { return HasLayoutNext; } + void setLayoutNext(DefinedAtom &Next) { + assert(!HasLayoutNext && "Atom already has layout-next constraint"); + HasLayoutNext = true; + Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0)); + } + DefinedAtom &getLayoutNext() { + assert(HasLayoutNext && "Atom does not have a layout-next constraint"); + DefinedAtom *Next = nullptr; + for (auto &E : edges()) + if (E.getKind() == Edge::LayoutNext) { + assert(E.getTarget().isDefined() && + "layout-next target atom must be a defined atom"); + Next = static_cast(&E.getTarget()); + break; + } + assert(Next && "Missing LayoutNext edge"); + return *Next; + } + + bool isCommon() const { return IsCommon; } + + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target, + Edge::AddendT Addend) { + assert(K != Edge::LayoutNext && + "Layout edges should be added via addLayoutNext"); + Edges.push_back(Edge(K, Offset, Target, Addend)); + } + + iterator_range edges() { + return make_range(Edges.begin(), Edges.end()); + } + size_t edges_size() const { return Edges.size(); } + bool edges_empty() const { return Edges.empty(); } + + unsigned getOrdinal() const { return Ordinal; } + +private: + void setCommon(uint64_t Size) { + assert(ContentPtr == 0 && "Atom already has content?"); + IsCommon = true; + setZeroFill(Size); + } + + EdgeVector Edges; + uint64_t Size = 0; + Section &Parent; + const char *ContentPtr = nullptr; + unsigned Ordinal = 0; + uint32_t Alignment = 0; +}; + +class AtomGraph { +private: + using SectionList = std::vector>; + using AddressToAtomMap = std::map; + using NamedAtomMap = DenseMap; + using ExternalAtomSet = DenseSet; + +public: + using external_atom_iterator = ExternalAtomSet::iterator; + + using section_iterator = pointee_iterator; + using const_section_iterator = pointee_iterator; + + template + class defined_atom_iterator_impl + : public iterator_facade_base< + defined_atom_iterator_impl, + std::forward_iterator_tag, T> { + public: + defined_atom_iterator_impl() = default; + + defined_atom_iterator_impl(SecItrT SI, SecItrT SE) + : SI(SI), SE(SE), + AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) { + moveToNextAtomOrEnd(); + } + + bool operator==(const defined_atom_iterator_impl &RHS) const { + return (SI == RHS.SI) && (AI == RHS.AI); + } + + T operator*() const { + assert(AI != SI->atoms().end() && "Dereferencing end?"); + return *AI; + } + + defined_atom_iterator_impl operator++() { + ++AI; + moveToNextAtomOrEnd(); + return *this; + } + + private: + void moveToNextAtomOrEnd() { + while (SI != SE && AI == SI->atoms().end()) { + ++SI; + if (SI == SE) + AI = Section::atom_iterator(); + else + AI = SI->atoms().begin(); + } + } + + SecItrT SI, SE; + AtomItrT AI; + }; + + using defined_atom_iterator = + defined_atom_iterator_impl; + + using const_defined_atom_iterator = + defined_atom_iterator_impl; + + AtomGraph(std::string Name, unsigned PointerSize, + support::endianness Endianness) + : Name(std::move(Name)), PointerSize(PointerSize), + Endianness(Endianness) {} + + /// Returns the name of this graph (usually the name of the original + /// underlying MemoryBuffer). + const std::string &getName() { return Name; } + + /// Returns the pointer size for use in this graph. + unsigned getPointerSize() const { return PointerSize; } + + /// Returns the endianness of atom-content in this graph. + support::endianness getEndianness() const { return Endianness; } + + /// Create a section with the given name, protection flags, and alignment. + Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot, + bool IsZeroFill) { + std::unique_ptr
Sec( + new Section(Name, Prot, Sections.size(), IsZeroFill)); + Sections.push_back(std::move(Sec)); + return *Sections.back(); + } + + /// Add an external atom representing an undefined symbol in this graph. + Atom &addExternalAtom(StringRef Name) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + Atom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); + new (A) Atom(Name); + ExternalAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + /// Add an external atom representing an absolute symbol. + Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + Atom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); + new (A) Atom(Name, Addr); + AbsoluteAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + /// Add an anonymous defined atom to the graph. + /// + /// Anonymous atoms have content but no name. They must have an address. + DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, + uint32_t Alignment) { + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Address, Alignment); + Parent.addAtom(*A); + getAddrToAtomMap()[A->getAddress()] = A; + return *A; + } + + /// Add a defined atom to the graph. + /// + /// Allocates and constructs a DefinedAtom instance with the given parent, + /// name, address, and alignment. + DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, + JITTargetAddress Address, uint32_t Alignment) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Name, Address, Alignment); + Parent.addAtom(*A); + getAddrToAtomMap()[A->getAddress()] = A; + NamedAtoms[Name] = A; + return *A; + } + + /// Add a common symbol atom to the graph. + /// + /// Adds a common-symbol atom to the graph with the given parent, name, + /// address, alignment and size. + DefinedAtom &addCommonAtom(Section &Parent, StringRef Name, + JITTargetAddress Address, uint32_t Alignment, + uint64_t Size) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Name, Address, Alignment); + A->setCommon(Size); + Parent.addAtom(*A); + NamedAtoms[Name] = A; + return *A; + } + + iterator_range sections() { + return make_range(section_iterator(Sections.begin()), + section_iterator(Sections.end())); + } + + iterator_range external_atoms() { + return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + } + + iterator_range absolute_atoms() { + return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end()); + } + + iterator_range defined_atoms() { + return make_range(defined_atom_iterator(Sections.begin(), Sections.end()), + defined_atom_iterator(Sections.end(), Sections.end())); + } + + iterator_range defined_atoms() const { + return make_range( + const_defined_atom_iterator(Sections.begin(), Sections.end()), + const_defined_atom_iterator(Sections.end(), Sections.end())); + } + + /// Returns the atom with the given name, which must exist in this graph. + Atom &getAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); + return *I->second; + } + + /// Returns the atom with the given name, which must exist in this graph and + /// be a DefinedAtom. + DefinedAtom &getDefinedAtomByName(StringRef Name) { + auto &A = getAtomByName(Name); + assert(A.isDefined() && "Atom is not a defined atom"); + return static_cast(A); + } + + /// Search for the given atom by name. + /// Returns the atom (if found) or an error (if no atom with this name + /// exists). + Expected findAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + return *I->second; + } + + /// Search for the given defined atom by name. + /// Returns the defined atom (if found) or an error (if no atom with this + /// name exists, or if one exists but is not a defined atom). + Expected findDefinedAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + if (!I->second->isDefined()) + return make_error("Atom " + Name + + " exists but is not a " + "defined atom"); + return static_cast(*I->second); + } + + /// Returns the atom covering the given address, or an error if no such atom + /// exists. + /// + /// Returns null if no atom exists at the given address. + DefinedAtom *getAtomByAddress(JITTargetAddress Address) { + refreshAddrToAtomCache(); + + // If there are no defined atoms, bail out early. + if (AddrToAtomCache->empty()) + return nullptr; + + // Find the atom *after* the given address. + auto I = AddrToAtomCache->upper_bound(Address); + + // If this address falls before any known atom, bail out. + if (I == AddrToAtomCache->begin()) + return nullptr; + + // The atom we're looking for is the one before the atom we found. + --I; + + // Otherwise range check the atom that was found. + assert(!I->second->getContent().empty() && "Atom content not set"); + if (Address >= I->second->getAddress() + I->second->getContent().size()) + return nullptr; + + return I->second; + } + + /// Like getAtomByAddress, but returns an Error if the given address is not + /// covered by an atom, rather than a null pointer. + Expected findAtomByAddress(JITTargetAddress Address) { + if (auto *DA = getAtomByAddress(Address)) + return *DA; + return make_error("No atom at address " + + formatv("{0:x16}", Address)); + } + + // Remove the given external atom from the graph. + void removeExternalAtom(Atom &A) { + assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom"); + assert(ExternalAtoms.count(&A) && "A is not in the external atoms set"); + ExternalAtoms.erase(&A); + A.~Atom(); + } + + /// Remove the given absolute atom from the graph. + void removeAbsoluteAtom(Atom &A) { + assert(A.isAbsolute() && "A is not an absolute atom"); + assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set"); + AbsoluteAtoms.erase(&A); + A.~Atom(); + } + + /// Remove the given defined atom from the graph. + void removeDefinedAtom(DefinedAtom &DA) { + if (AddrToAtomCache) { + assert(AddrToAtomCache->count(DA.getAddress()) && + "Cache exists, but does not contain atom"); + AddrToAtomCache->erase(DA.getAddress()); + } + if (DA.hasName()) { + assert(NamedAtoms.count(DA.getName()) && "Named atom not in map"); + NamedAtoms.erase(DA.getName()); + } + DA.getSection().removeAtom(DA); + DA.~DefinedAtom(); + } + + /// Invalidate the atom-to-address map. + void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + + /// Dump the graph. + /// + /// If supplied, the EdgeKindToName function will be used to name edge + /// kinds in the debug output. Otherwise raw edge kind numbers will be + /// displayed. + void dump(raw_ostream &OS, + std::function EdegKindToName = + std::function()); + +private: + AddressToAtomMap &getAddrToAtomMap() { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + const AddressToAtomMap &getAddrToAtomMap() const { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + void refreshAddrToAtomCache() const { + if (!AddrToAtomCache) { + AddrToAtomCache = AddressToAtomMap(); + for (auto *DA : defined_atoms()) + (*AddrToAtomCache)[DA->getAddress()] = const_cast(DA); + } + } + + // Put the BumpPtrAllocator first so that we don't free any of the atoms in + // it until all of their destructors have been run. + BumpPtrAllocator AtomAllocator; + + std::string Name; + unsigned PointerSize; + support::endianness Endianness; + SectionList Sections; + NamedAtomMap NamedAtoms; + ExternalAtomSet ExternalAtoms; + ExternalAtomSet AbsoluteAtoms; + mutable Optional AddrToAtomCache; +}; + +/// A function for mutating AtomGraphs. +using AtomGraphPassFunction = std::function; + +/// A list of atom graph passes. +using AtomGraphPassList = std::vector; + +/// An atom graph pass configuration, consisting of a list of pre-prune, +/// post-prune, and post-fixup passes. +struct PassConfiguration { + + /// Pre-prune passes. + /// + /// These passes are called on the graph after it is built, and before any + /// atoms have been pruned. + /// + /// Notable use cases: Marking atoms live or should-discard. + AtomGraphPassList PrePrunePasses; + + /// Post-prune passes. + /// + /// These passes are called on the graph after dead and should-discard atoms + /// have been removed, but before fixups are applied. + /// + /// Notable use cases: Building GOT, stub, and TLV atoms. + AtomGraphPassList PostPrunePasses; + + /// Post-fixup passes. + /// + /// These passes are called on the graph after atom contents has been copied + /// to working memory, and fixups applied. + /// + /// Notable use cases: Testing and validation. + AtomGraphPassList PostFixupPasses; +}; + +/// A JITLinkMemoryManager that allocates in-process memory. +class InProcessMemoryManager : public JITLinkMemoryManager { +public: + Expected> + allocate(const SegmentsRequestMap &Request) override; +}; + +/// A map of symbol names to resolved addresses. +using AsyncLookupResult = DenseMap; + +/// A function to call with a resolved symbol map (See AsyncLookupResult) or an +/// error if resolution failed. +using JITLinkAsyncLookupContinuation = + std::function LR)>; + +/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) +/// for the given symbols, calling the given continuation with either the result +/// (if the lookup succeeds), or an error (if the lookup fails). +using JITLinkAsyncLookupFunction = + std::function &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation)>; + +/// Holds context for a single jitLink invocation. +class JITLinkContext { +public: + /// Destroy a JITLinkContext. + virtual ~JITLinkContext(); + + /// Return the MemoryManager to be used for this link. + virtual JITLinkMemoryManager &getMemoryManager() = 0; + + /// Returns a StringRef for the object buffer. + /// This method can not be called once takeObjectBuffer has been called. + virtual MemoryBufferRef getObjectBuffer() const = 0; + + /// Notify this context that linking failed. + /// Called by JITLink if linking cannot be completed. + virtual void notifyFailed(Error Err) = 0; + + /// Called by JITLink to resolve external symbols. This method is passed a + /// lookup continutation which it must call with a result to continue the + /// linking process. + virtual void lookup(const DenseSet &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) = 0; + + /// Called by JITLink once all defined atoms in the graph have been assigned + /// their final memory locations in the target process. At this point he + /// atom graph can be, inspected to build a symbol table however the atom + /// content will not generally have been copied to the target location yet. + virtual void notifyResolved(AtomGraph &G) = 0; + + /// Called by JITLink to notify the context that the object has been + /// finalized (i.e. emitted to memory and memory permissions set). If all of + /// this objects dependencies have also been finalized then the code is ready + /// to run. + virtual void + notifyFinalized(std::unique_ptr A) = 0; + + /// Called by JITLink prior to linking to determine whether default passes for + /// the target should be added. The default implementation returns true. + /// If subclasses override this method to return false for any target then + /// they are required to fully configure the pass pipeline for that target. + virtual bool shouldAddDefaultTargetPasses(const Triple &TT) const; + + /// Returns the mark-live pass to be used for this link. If no pass is + /// returned (the default) then the target-specific linker implementation will + /// choose a conservative default (usually marking all atoms live). + /// This function is only called if shouldAddDefaultTargetPasses returns true, + /// otherwise the JITContext is responsible for adding a mark-live pass in + /// modifyPassConfig. + virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const; + + /// Called by JITLink to modify the pass pipeline prior to linking. + /// The default version performs no modification. + virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config); +}; + +/// Marks all atoms in a graph live. This can be used as a default, conservative +/// mark-live implementation. +Error markAllAtomsLive(AtomGraph &G); + +/// Basic JITLink implementation. +/// +/// This function will use sensible defaults for GOT and Stub handling. +void jitLink(std::unique_ptr Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H Index: llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h +++ llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h @@ -0,0 +1,42 @@ +//===----- JITLink_EHFrameSupport.h - JITLink eh-frame utils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORT_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORT_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace jitlink { + +/// Registers all FDEs in the given eh-frame section with the current process. +Error registerEHFrameSection(const void *EHFrameSectionAddr); + +/// Deregisters all FDEs in the given eh-frame section with the current process. +Error deregisterEHFrameSection(const void *EHFrameSectionAddr); + +/// Creates a pass that records the address of the EH frame section. If no +/// eh-frame section is found, it will set EHFrameAddr to zero. +/// +/// Authors of JITLinkContexts can use this function to register a post-fixup +/// pass that records the address of the eh-frame section. This address can +/// be used after finalization to register and deregister the frame. +AtomGraphPassFunction createEHFrameRecorderPass(const Triple &TT, + JITTargetAddress &EHFrameAddr); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORT_H Index: llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO.h +++ llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO.h @@ -0,0 +1,30 @@ +//===--- JITLink_MachO.h - Generic JIT link function for MachO --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic jit-link functions for MachO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// jit-link the given ObjBuffer, which must be a MachO object file. +/// +/// Uses conservative defaults for GOT and stub handling based on the target +/// platform. +void jitLink_MachO(std::unique_ptr Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H Index: llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h +++ llvm/trunk/include/llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h @@ -0,0 +1,63 @@ +//===--- JITLink_MachO_x86_64.h - JIT link functions for MachO --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/x86-64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_x86_64_Edges { + +enum MachOX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Pointer64, + Pointer64Anon, + PCRel32, + PCRel32Minus1, + PCRel32Minus2, + PCRel32Minus4, + PCRel32Anon, + PCRel32Minus1Anon, + PCRel32Minus2Anon, + PCRel32Minus4Anon, + PCRel32GOTLoad, + PCRel32GOT, + PCRel32TLV, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_x86_64_Edges + +/// jit-link the given object buffer, which must be a MachO x86-64 object file. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void jitLink_MachO_x86_64(std::unique_ptr Ctx); + +/// Return the string name of the given MachO x86-64 edge kind. +StringRef getMachOX86RelocationKindName(Edge::Kind R); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H Index: llvm/trunk/include/llvm/ExecutionEngine/JITSymbol.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/JITSymbol.h +++ llvm/trunk/include/llvm/ExecutionEngine/JITSymbol.h @@ -55,7 +55,7 @@ class JITSymbolFlags { public: using UnderlyingType = uint8_t; - using TargetFlagsType = uint64_t; + using TargetFlagsType = uint8_t; enum FlagNames : UnderlyingType { None = 0, @@ -83,7 +83,7 @@ /// Construct a JITSymbolFlags instance from the given flags and target /// flags. JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) - : Flags(Flags), TargetFlags(TargetFlags) {} + : TargetFlags(TargetFlags), Flags(Flags) {} /// Implicitly convert to bool. Returs true if any flag is set. explicit operator bool() const { return Flags != None || TargetFlags != 0; } @@ -167,8 +167,8 @@ fromObjectSymbol(const object::SymbolRef &Symbol); private: - FlagNames Flags = None; TargetFlagsType TargetFlags = 0; + FlagNames Flags = None; }; inline JITSymbolFlags operator&(const JITSymbolFlags &LHS, Index: llvm/trunk/include/llvm/ExecutionEngine/Orc/Core.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/Orc/Core.h +++ llvm/trunk/include/llvm/ExecutionEngine/Orc/Core.h @@ -174,7 +174,7 @@ /// Note: The returned flags may have transient flags (Lazy, Materializing) /// set. These should be stripped with JITSymbolFlags::stripTransientFlags /// before using. - const SymbolFlagsMap &getSymbols() { return SymbolFlags; } + const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } /// Returns the names of any symbols covered by this /// MaterializationResponsibility object that have queries pending. This Index: llvm/trunk/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ llvm/trunk/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -217,25 +217,26 @@ /// Create a DynamicLibrarySearchGenerator that searches for symbols in the /// given sys::DynamicLibrary. + /// /// If the Allow predicate is given then only symbols matching the predicate - /// will be searched for in the DynamicLibrary. If the predicate is not given - /// then all symbols will be searched for. - DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, const DataLayout &DL, + /// will be searched for. If the predicate is not given then all symbols will + /// be searched for. + DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. static Expected - Load(const char *FileName, const DataLayout &DL, + Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. static Expected - GetForCurrentProcess(const DataLayout &DL, + GetForCurrentProcess(char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()) { - return Load(nullptr, DL, std::move(Allow)); + return Load(nullptr, GlobalPrefix, std::move(Allow)); } SymbolNameSet operator()(JITDylib &JD, const SymbolNameSet &Names); Index: llvm/trunk/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h =================================================================== --- llvm/trunk/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ llvm/trunk/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -0,0 +1,134 @@ +//===-- ObjectLinkingLayer.h - JITLink-based jit linking layer --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition for an JITLink-based, in-process object linking +// layer. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Support/Error.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { + +namespace object { +class ObjectFile; +} // namespace object + +namespace orc { + +class ObjectLinkingLayerJITLinkContext; + +class ObjectLinkingLayer : public ObjectLayer { + friend class ObjectLinkingLayerJITLinkContext; + +public: + /// Function object for receiving object-loaded notifications. + using NotifyLoadedFunction = std::function; + + /// Function object for receiving finalization notifications. + using NotifyEmittedFunction = std::function; + + /// Function object for modifying PassConfiguration objects. + using ModifyPassConfigFunction = + std::function; + + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, + /// and NotifyEmitted functors. + ObjectLinkingLayer( + ExecutionSession &ES, jitlink::JITLinkMemoryManager &MemMgr, + NotifyLoadedFunction NotifyLoaded = NotifyLoadedFunction(), + NotifyEmittedFunction NotifyEmitted = NotifyEmittedFunction(), + ModifyPassConfigFunction ModifyPassConfig = ModifyPassConfigFunction()); + + /// Emit the object. + void emit(MaterializationResponsibility R, + std::unique_ptr O) override; + + /// Instructs this ObjectLinkingLayer instance to override the symbol flags + /// found in the AtomGraph with the flags supplied by the + /// MaterializationResponsibility instance. This is a workaround to support + /// symbol visibility in COFF, which does not use the libObject's + /// SF_Exported flag. Use only when generating / adding COFF object files. + /// + /// FIXME: We should be able to remove this if/when COFF properly tracks + /// exported symbols. + ObjectLinkingLayer & + setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags) { + this->OverrideObjectFlags = OverrideObjectFlags; + return *this; + } + + /// If set, this ObjectLinkingLayer instance will claim responsibility + /// for any symbols provided by a given object file that were not already in + /// the MaterializationResponsibility instance. Setting this flag allows + /// higher-level program representations (e.g. LLVM IR) to be added based on + /// only a subset of the symbols they provide, without having to write + /// intervening layers to scan and add the additional symbols. This trades + /// diagnostic quality for convenience however: If all symbols are enumerated + /// up-front then clashes can be detected and reported early (and usually + /// deterministically). If this option is set, clashes for the additional + /// symbols may not be detected until late, and detection may depend on + /// the flow of control through JIT'd code. Use with care. + ObjectLinkingLayer & + setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) { + this->AutoClaimObjectSymbols = AutoClaimObjectSymbols; + return *this; + } + +private: + using AllocPtr = std::unique_ptr; + + class ObjectResources { + public: + ObjectResources() = default; + ObjectResources(AllocPtr Alloc, JITTargetAddress EHFrameAddr); + ObjectResources(ObjectResources &&Other); + ObjectResources &operator=(ObjectResources &&Other); + ~ObjectResources(); + + private: + AllocPtr Alloc; + JITTargetAddress EHFrameAddr = 0; + }; + + void notifyFinalized(ObjectResources OR) { + ObjResources.push_back(std::move(OR)); + } + + mutable std::mutex LayerMutex; + jitlink::JITLinkMemoryManager &MemMgr; + NotifyLoadedFunction NotifyLoaded; + NotifyEmittedFunction NotifyEmitted; + ModifyPassConfigFunction ModifyPassConfig; + bool OverrideObjectFlags = false; + bool AutoClaimObjectSymbols = false; + std::vector ObjResources; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H Index: llvm/trunk/include/llvm/Support/Memory.h =================================================================== --- llvm/trunk/include/llvm/Support/Memory.h +++ llvm/trunk/include/llvm/Support/Memory.h @@ -18,6 +18,10 @@ #include namespace llvm { + +// Forward declare raw_ostream: it is used for debug dumping below. +class raw_ostream; + namespace sys { /// This class encapsulates the notion of a memory block which has an address @@ -141,7 +145,14 @@ MemoryBlock M; }; -} -} +#ifndef NDEBUG + /// Debugging output for Memory::ProtectionFlags. + raw_ostream &operator<<(raw_ostream &OS, const Memory::ProtectionFlags &PF); + + /// Debugging output for MemoryBlock. + raw_ostream &operator<<(raw_ostream &OS, const MemoryBlock &MB); +#endif // ifndef NDEBUG + } // end namespace sys + } // end namespace llvm #endif Index: llvm/trunk/lib/ExecutionEngine/CMakeLists.txt =================================================================== --- llvm/trunk/lib/ExecutionEngine/CMakeLists.txt +++ llvm/trunk/lib/ExecutionEngine/CMakeLists.txt @@ -19,6 +19,7 @@ endif() add_subdirectory(Interpreter) +add_subdirectory(JITLink) add_subdirectory(MCJIT) add_subdirectory(Orc) add_subdirectory(RuntimeDyld) Index: llvm/trunk/lib/ExecutionEngine/JITLink/CMakeLists.txt =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ llvm/trunk/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -0,0 +1,11 @@ +add_llvm_library(LLVMJITLink + JITLink.cpp + JITLinkGeneric.cpp + JITLink_EHFrameSupport.cpp + JITLink_MachO.cpp + JITLink_MachO_x86_64.cpp + MachOAtomGraphBuilder.cpp + + DEPENDS + intrinsics_gen + ) Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLink.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLink.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,261 @@ +//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; + +JITLinkMemoryManager::Allocation::~Allocation() = default; + +const StringRef getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + case Edge::LayoutNext: + return "Layout-Next"; + default: + llvm_unreachable("Unrecognized relocation kind"); + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { + OS << "<"; + if (A.getName().empty()) + OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + else + OS << A.getName(); + OS << " ["; + if (A.isDefined()) { + auto &DA = static_cast(A); + OS << " section=" << DA.getSection().getName(); + if (DA.isLive()) + OS << " live"; + if (DA.shouldDiscard()) + OS << " should-discard"; + } else + OS << " external"; + OS << " ]>"; + return OS; +} + +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) + << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName + << " -> " << E.getTarget() << " + " << E.getAddend(); +} + +Section::~Section() { + for (auto *DA : DefinedAtoms) + DA->~DefinedAtom(); +} + +void AtomGraph::dump(raw_ostream &OS, + std::function EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Defined atoms:\n"; + for (auto *DA : defined_atoms()) { + OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + << "\n"; + for (auto &E : DA->edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, *DA, E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, *DA, E, EdgeNumberString); + } + OS << "\n"; + } + } + + OS << "Absolute atoms:\n"; + for (auto *A : absolute_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; + + OS << "External atoms:\n"; + for (auto *A : external_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; +} + +Expected> +InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast(SegBlocks[Seg].base()), + SegBlocks[Seg].size()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return reinterpret_cast(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + for (auto &KV : SegBlocks) + if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + for (auto &KV : Request) { + auto &Seg = KV.second; + + if (Seg.getContentAlignment() > sys::Process::getPageSize()) + return make_error("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + if (sys::Process::getPageSize() % Seg.getContentAlignment() != 0) + return make_error("Page size is not a multiple of " + "alignment", + inconvertibleErrorCode()); + + uint64_t ZeroFillStart = + alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); + uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + + std::error_code EC; + auto SegMem = + sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Zero out the zero-fill memory. + bzero(static_cast(SegMem.base()) + ZeroFillStart, + Seg.getZeroFillSize()); + + // Record the block for this segment. + Blocks[KV.first] = std::move(SegMem); + } + return std::unique_ptr( + new IPMMAlloc(std::move(Blocks))); +} + +JITLinkContext::~JITLinkContext() {} + +bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { + return true; +} + +AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return AtomGraphPassFunction(); +} + +Error JITLinkContext::modifyPassConfig(const Triple &TT, + PassConfiguration &Config) { + return Error::success(); +} + +Error markAllAtomsLive(AtomGraph &G) { + for (auto *DA : G.defined_atoms()) + DA->setLive(true); + return Error::success(); +} + +void jitLink(std::unique_ptr Ctx) { + auto Magic = identify_magic(Ctx->getObjectBuffer().getBuffer()); + switch (Magic) { + case file_magic::macho_object: + return jitLink_MachO(std::move(Ctx)); + default: + Ctx->notifyFailed(make_error("Unsupported file format")); + }; +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.h =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,246 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr Ctx, PassConfiguration Passes) + : Ctx(std::move(Ctx)), Passes(std::move(Passes)) { + assert(this->Ctx && "Ctx can not be null"); + } + + virtual ~JITLinkerBase(); + +protected: + struct SegmentLayout { + using SectionAtomsList = std::vector; + struct SectionLayout { + SectionLayout(Section &S) : S(&S) {} + + Section *S; + SectionAtomsList Atoms; + }; + + using SectionLayoutList = std::vector; + + SectionLayoutList ContentSections; + SectionLayoutList ZeroFillSections; + }; + + using SegmentLayoutMap = DenseMap; + + // Phase 1: + // 1.1: Build atom graph + // 1.2: Run pre-prune passes + // 1.2: Prune graph + // 1.3: Run post-prune passes + // 1.4: Sort atoms into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up atom contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr Self, + Expected LookupResult); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr Self, Error Err); + + // Build a graph from the given object buffer. + // To be implemented by the client. + virtual Expected> + buildGraph(MemoryBufferRef ObjBuffer) = 0; + + // For debug dumping of the atom graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + +private: + // Run all passes in the given pass list, bailing out immediately if any pass + // returns an error. + Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + + // Copy atom contents and apply relocations. + // Implemented in JITLinker. + virtual Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; + + void layOutAtoms(); + Error allocateSegments(const SegmentLayoutMap &Layout); + DenseSet getExternalSymbolNames() const; + void applyLookupResult(AsyncLookupResult LR); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr Ctx; + PassConfiguration Passes; + std::unique_ptr G; + SegmentLayoutMap Layout; + std::unique_ptr Alloc; +}; + +template class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template static void link(ArgTs &&... Args) { + auto L = llvm::make_unique(std::forward(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast(*this); + } + + Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = Alloc.getWorkingMemory( + static_cast(Prot)); + char *LastAtomEnd = SegMem.data(); + char *AtomDataPtr = nullptr; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto &SI : SegLayout.ContentSections) { + LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + for (auto *DA : SI.Atoms) { + AtomDataPtr = LastAtomEnd; + + // Align. + AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " + << (const void *)AtomDataPtr << " to meet alignment of " + << DA->getAlignment() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastAtomEnd != AtomDataPtr) + dbgs() << " Zero padding from " << (const void *)LastAtomEnd + << " to " << (const void *)AtomDataPtr << "\n"; + }); + while (LastAtomEnd != AtomDataPtr) + *LastAtomEnd++ = 0; + + // Copy initial atom content. + LLVM_DEBUG({ + dbgs() << " Copying atom " << *DA << " content, " + << DA->getContent().size() << " bytes, from " + << (const void *)DA->getContent().data() << " to " + << (const void *)AtomDataPtr << "\n"; + }); + memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); + + // Copy atom data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : DA->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) + return Err; + } + + // Point the atom's content to the fixed up buffer. + DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); + + // Update atom end pointer. + LastAtomEnd = AtomDataPtr + DA->getContent().size(); + } + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastAtomEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastAtomEnd != SegMem.data() + SegMem.size()) + *LastAtomEnd++ = 0; + } + + return Error::success(); + } +}; + +/// Dead strips and replaces discarded definitions with external atoms. +/// +/// Finds the set of nodes reachable from any node initially marked live +/// (nodes marked should-discard are treated as not live, even if they are +/// reachable). All nodes not marked as live at the end of this process, +/// are deleted. Nodes that are live, but marked should-discard are replaced +/// with external atoms and all edges to them are re-written. +void prune(AtomGraph &G); + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,464 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" +#include "JITLink_EHFrameSupportImpl.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr Self) { + + // Build the atom graph. + if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) + G = std::move(*GraphOrErr); + else + return Ctx->notifyFailed(GraphOrErr.takeError()); + assert(G && "Graph should have been created by buildGraph above"); + + // Prune and optimize the graph. + if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dumpGraph(dbgs()); + }); + + prune(*G); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dumpGraph(dbgs()); + }); + + // Run post-pruning passes. + if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // Sort atoms into segments. + layOutAtoms(); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return Ctx->notifyFailed(std::move(Err)); + + // Notify client that the defined atoms have been assigned addresses. + Ctx->notifyResolved(*G); + + auto ExternalSymbols = getExternalSymbolNames(); + + // We're about to hand off ownership of ourself to the continuation. Grab a + // pointer to the context so that we can call it to initiate the lookup. + // + // FIXME: Once callee expressions are defined to be sequenced before argument + // expressions (c++17) we can simplify all this to: + // + // Ctx->lookup(std::move(UnresolvedExternals), + // [Self=std::move(Self)](Expected Result) { + // Self->linkPhase2(std::move(Self), std::move(Result)); + // }); + // + // FIXME: Use move capture once we have c++14. + auto *TmpCtx = Ctx.get(); + auto *UnownedSelf = Self.release(); + auto Phase2Continuation = + [UnownedSelf](Expected LookupResult) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); + }; + TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr Self, + Expected LR) { + // If the lookup failed, bail out. + if (!LR) + return Ctx->notifyFailed(LR.takeError()); + + // Assign addresses to external atoms. + applyLookupResult(*LR); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + // Copy atom content to working memory and fix up. + if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr Self, Error Err) { + if (Err) + return Ctx->notifyFailed(std::move(Err)); + Ctx->notifyFinalized(std::move(Alloc)); +} + +Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { + for (auto &P : Passes) + if (auto Err = P(G)) + return Err; + return Error::success(); +} + +void JITLinkerBase::layOutAtoms() { + // Group sections by protections, and whether or not they're zero-fill. + for (auto &S : G->sections()) { + + // Skip empty sections. + if (S.atoms_empty()) + continue; + + auto &SL = Layout[S.getProtectionFlags()]; + if (S.isZeroFill()) + SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + else + SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + } + + // Sort sections within the layout by ordinal. + { + auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, + const SegmentLayout::SectionLayout &RHS) { + return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + }; + for (auto &KV : Layout) { + auto &SL = KV.second; + std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), + CompareByOrdinal); + std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), + CompareByOrdinal); + } + } + + // Add atoms to the sections. + for (auto &KV : Layout) { + auto &SL = KV.second; + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { + for (auto &SI : *SIList) { + std::vector LayoutHeads; + LayoutHeads.reserve(SI.S->atoms_size()); + + // First build the list of layout-heads (i.e. "heads" of layout-next + // chains). + DenseSet AlreadyLayedOut; + for (auto *DA : SI.S->atoms()) { + if (AlreadyLayedOut.count(DA)) + continue; + LayoutHeads.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + AlreadyLayedOut.insert(&Next); + DA = &Next; + } + } + + // Now sort the list of layout heads by address. + std::sort(LayoutHeads.begin(), LayoutHeads.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Now populate the SI.Atoms field by appending each of the chains. + for (auto *DA : LayoutHeads) { + SI.Atoms.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + SI.Atoms.push_back(&Next); + DA = &Next; + } + } + } + } + } + + LLVM_DEBUG({ + dbgs() << "Segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast(KV.first) << ":\n"; + auto &SL = KV.second; + for (auto &SIEntry : + {std::make_pair(&SL.ContentSections, "content sections"), + std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { + auto &SIList = *SIEntry.first; + dbgs() << " " << SIEntry.second << ":\n"; + for (auto &SI : SIList) { + dbgs() << " " << SI.S->getName() << ":\n"; + for (auto *DA : SI.Atoms) + dbgs() << " " << *DA << "\n"; + } + } + } + }); +} + +Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + // Calculate segment content size. + size_t SegContentSize = 0; + for (auto &SI : SegLayout.ContentSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + for (auto *DA : SI.Atoms) { + SegContentSize = alignTo(SegContentSize, DA->getAlignment()); + SegContentSize += DA->getSize(); + } + } + + // Get segment content alignment. + unsigned SegContentAlign = 1; + if (!SegLayout.ContentSections.empty()) + SegContentAlign = + SegLayout.ContentSections.front().Atoms.front()->getAlignment(); + + // Calculate segment zero-fill size. + uint64_t SegZeroFillSize = 0; + for (auto &SI : SegLayout.ZeroFillSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + for (auto *DA : SI.Atoms) { + SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); + SegZeroFillSize += DA->getSize(); + } + } + + // Calculate segment zero-fill alignment. + uint32_t SegZeroFillAlign = 1; + if (!SegLayout.ZeroFillSections.empty()) + SegZeroFillAlign = + SegLayout.ZeroFillSections.front().Atoms.front()->getAlignment(); + + if (SegContentSize == 0) + SegContentAlign = SegZeroFillAlign; + + if (SegContentAlign % SegZeroFillAlign != 0) + return make_error("First content atom alignment does not " + "accommodate first zero-fill atom " + "alignment"); + + Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, + SegZeroFillAlign}; + + LLVM_DEBUG({ + dbgs() << (&KV == &*Layout.begin() ? "" : "; ") + << static_cast(Prot) << ": " + << SegContentSize << " content bytes (alignment " + << SegContentAlign << ") + " << SegZeroFillSize + << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + }); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << "JIT linker got working memory:\n"; + for (auto &KV : Layout) { + auto Prot = static_cast(KV.first); + dbgs() << " " << Prot << ": " + << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + } + }); + + // Update atom target addresses. + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SL = KV.second; + + JITTargetAddress AtomTargetAddr = + Alloc->getTargetMemory(static_cast(Prot)); + + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) + for (auto &SI : *SIList) + for (auto *DA : SI.Atoms) { + AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); + DA->setAddress(AtomTargetAddr); + AtomTargetAddr += DA->getSize(); + } + } + + return Error::success(); +} + +DenseSet JITLinkerBase::getExternalSymbolNames() const { + // Identify unresolved external atoms. + DenseSet UnresolvedExternals; + for (auto *DA : G->external_atoms()) { + assert(DA->getAddress() == 0 && + "External has already been assigned an address"); + assert(DA->getName() != StringRef() && DA->getName() != "" && + "Externals must be named"); + UnresolvedExternals.insert(DA->getName()); + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto &KV : Result) { + Atom &A = G->getAtomByName(KV.first); + assert(A.getAddress() == 0 && "Atom already resolved"); + A.setAddress(KV.second.getAddress()); + } + + assert(llvm::all_of(G->external_atoms(), + [](Atom *A) { return A->getAddress() != 0; }) && + "All atoms should have been resolved by this point"); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void prune(AtomGraph &G) { + std::vector Worklist; + DenseMap> EdgesToUpdate; + + // Build the initial worklist from all atoms initially live. + for (auto *DA : G.defined_atoms()) { + if (!DA->isLive() || DA->shouldDiscard()) + continue; + + for (auto &E : DA->edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Propagate live flags to all atoms reachable from the initial live set. + while (!Worklist.empty()) { + DefinedAtom &NextLive = *Worklist.back(); + Worklist.pop_back(); + + assert(!NextLive.shouldDiscard() && + "should-discard nodes should never make it into the worklist"); + + // If this atom has already been marked as live, or is marked to be + // discarded, then skip it. + if (NextLive.isLive()) + continue; + + // Otherwise set it as live and add any non-live atoms that it points to + // to the worklist. + NextLive.setLive(true); + + for (auto &E : NextLive.edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Collect atoms to remove, then remove them from the graph. + std::vector AtomsToRemove; + for (auto *DA : G.defined_atoms()) + if (DA->shouldDiscard() || !DA->isLive()) + AtomsToRemove.push_back(DA); + + LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); + for (auto *DA : AtomsToRemove) { + LLVM_DEBUG(dbgs() << " " << *DA << "... "); + + // Check whether we need to replace this atom with an external atom. + // + // We replace if all of the following hold: + // (1) The atom is marked should-discard, + // (2) it is live, and + // (3) it has edges pointing to it. + // + // Otherwise we simply delete the atom. + bool ReplaceWithExternal = DA->isLive() && DA->shouldDiscard(); + std::vector *EdgesToUpdateForDA = nullptr; + if (ReplaceWithExternal) { + auto ETUItr = EdgesToUpdate.find(DA); + if (ETUItr == EdgesToUpdate.end()) + ReplaceWithExternal = false; + else + EdgesToUpdateForDA = &ETUItr->second; + } + + G.removeDefinedAtom(*DA); + + if (ReplaceWithExternal) { + assert(EdgesToUpdateForDA && + "Replacing atom: There should be edges to update"); + + auto &ExternalReplacement = G.addExternalAtom(DA->getName()); + for (auto *EdgeToUpdate : *EdgesToUpdateForDA) + EdgeToUpdate->setTarget(ExternalReplacement); + LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); + } else + LLVM_DEBUG(dbgs() << "deleted\n"); + } + + // Finally, discard any absolute symbols that were marked should-discard. + { + std::vector AbsoluteAtomsToRemove; + for (auto *A : G.absolute_atoms()) + if (A->shouldDiscard() || A->isLive()) + AbsoluteAtomsToRemove.push_back(A); + for (auto *A : AbsoluteAtomsToRemove) + G.removeAbsoluteAtom(*A); + } +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp @@ -0,0 +1,537 @@ +//===-------- JITLink_EHFrameSupport.cpp - JITLink eh-frame utils ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "JITLink_EHFrameSupportImpl.h" + +#include "llvm/BinaryFormat/Dwarf.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), + EHFrameAddress(EHFrameAddress), + EHFrameReader(EHFrameContent, G.getEndianness()), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + +Error EHFrameParser::atomize() { + while (!EHFrameReader.empty()) { + size_t RecordOffset = EHFrameReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << "Processing eh-frame record at " + << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) + << " (offset " << RecordOffset << ")\n"; + }); + + size_t CIELength = 0; + uint32_t CIELengthField; + if (auto Err = EHFrameReader.readInteger(CIELengthField)) + return Err; + + // Process CIE length/extended-length fields to build the atom. + // + // The value of these fields describe the length of the *rest* of the CIE + // (not including data up to the end of the field itself) so we have to + // bump CIELength to include the data up to the end of the field: 4 bytes + // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. + if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + break; + + // If the regular length field's value is 0xffffffff, use extended length. + if (CIELengthField == 0xffffffff) { + uint64_t CIEExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + return Err; + if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + if (CIEExtendedLengthField + 12 > std::numeric_limits::max()) + return make_error("CIE record too large to process"); + CIELength = CIEExtendedLengthField + 12; + } else { + if (CIELengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + CIELength = CIELengthField + 4; + } + + LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); + + // Add an atom for this record. + CurRecordAtom = &G.addAnonymousAtom( + EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); + CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + + // Read the CIE Pointer. + size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); + uint32_t CIEPointer; + if (auto Err = EHFrameReader.readInteger(CIEPointer)) + return Err; + + // Based on the CIE pointer value, parse this as a CIE or FDE record. + if (CIEPointer == 0) { + if (auto Err = processCIE()) + return Err; + } else { + if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + return Err; + } + + EHFrameReader.setOffset(RecordOffset + CIELength); + } + + return Error::success(); +} + +Expected +EHFrameParser::parseAugmentationString() { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + } + + return std::move(AugInfo); +} + +Expected EHFrameParser::readAbsolutePointer() { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = EHFrameReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = EHFrameReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Error EHFrameParser::processCIE() { + // Use the dwarf namespace for convenient access to pointer encoding + // constants. + using namespace dwarf; + + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + + /// Reset state for the new CIE. + MostRecentCIE = CurRecordAtom; + LSDAFieldPresent = false; + + uint8_t Version = 0; + if (auto Err = EHFrameReader.readInteger(Version)) + return Err; + + if (Version != 0x01) + return make_error("Bad CIE version " + Twine(Version) + + " (should be 0x01) in eh-frame"); + + auto AugInfo = parseAugmentationString(); + if (!AugInfo) + return AugInfo.takeError(); + + // Skip the EH Data field if present. + if (AugInfo->EHDataFieldPresent) + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + // Read and sanity check the code alignment factor. + { + uint64_t CodeAlignmentFactor = 0; + if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + return Err; + if (CodeAlignmentFactor != 1) + return make_error("Unsupported CIE code alignment factor " + + Twine(CodeAlignmentFactor) + + " (expected 1)"); + } + + // Read and sanity check the data alignment factor. + { + int64_t DataAlignmentFactor = 0; + if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + return Err; + if (DataAlignmentFactor != -8) + return make_error("Unsupported CIE data alignment factor " + + Twine(DataAlignmentFactor) + + " (expected -8)"); + } + + // Skip the return address register field. + if (auto Err = EHFrameReader.skip(1)) + return Err; + + uint64_t AugmentationDataLength = 0; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': { + LSDAFieldPresent = true; + uint8_t LSDAPointerEncoding; + if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + return Err; + if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error( + "Unsupported LSDA pointer encoding " + + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + case 'P': { + uint8_t PersonalityPointerEncoding = 0; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + return Err; + if (PersonalityPointerEncoding != + (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) + return make_error( + "Unspported personality pointer " + "encoding " + + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + uint32_t PersonalityPointerAddress; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + return Err; + break; + } + case 'R': { + uint8_t FDEPointerEncoding; + if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + return Err; + if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error( + "Unsupported FDE address pointer " + "encoding " + + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + default: + llvm_unreachable("Invalid augmentation string field"); + } + } + + if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error("Read past the end of the augmentation " + "data while parsing fields"); + + return Error::success(); +} + +Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + // Sanity check the CIE pointer: if this is an FDE it must be proceeded by + // a CIE. + if (MostRecentCIE == nullptr) + return make_error("__eh_frame must start with CIE, not " + "FDE"); + + LLVM_DEBUG({ + dbgs() << " CIE pointer: " + << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; + }); + + // Verify that this FDE's CIE pointer points to the most recent CIE entry. + if (CIEPointerAddress - CIEPointer != MostRecentCIE->getAddress()) + return make_error("__eh_frame FDE's CIE Pointer does not " + "point at the most recent CIE"); + + // The CIEPointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToCIERelocKind, + CIEPointerAddress - CurRecordAtom->getAddress(), + *MostRecentCIE, 0); + + // Read and sanity check the PC-start pointer and size. + JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + + auto PCBeginDelta = readAbsolutePointer(); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + + JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + }); + + auto *TargetAtom = G.getAtomByAddress(PCBegin); + + if (!TargetAtom) + return make_error("FDE PC-begin " + + formatv("{0:x16}", PCBegin) + + " does not point at atom"); + + if (TargetAtom->getAddress() != PCBegin) + return make_error( + "FDE PC-begin " + formatv("{0:x16}", PCBegin) + + " does not point to start of atom at " + + formatv("{0:x16}", TargetAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + + // The PC-start pointer and size look good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + PCBeginAddress - CurRecordAtom->getAddress(), + *TargetAtom, 0); + + // Add a keep-alive relocation from the function to the FDE to ensure it is + // not dead stripped. + TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + + // Skip over the PC range size field. + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + if (LSDAFieldPresent) { + uint64_t AugmentationDataSize; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + return Err; + if (AugmentationDataSize != G.getPointerSize()) + return make_error("Unexpected FDE augmentation data size " + "(expected " + + Twine(G.getPointerSize()) + ", got " + + Twine(AugmentationDataSize) + ")"); + JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto LSDADelta = readAbsolutePointer(); + if (!LSDADelta) + return LSDADelta.takeError(); + + JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + + auto *LSDAAtom = G.getAtomByAddress(LSDA); + + if (!LSDAAtom) + return make_error("FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point at atom"); + + if (LSDAAtom->getAddress() != LSDA) + return make_error( + "FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point to start of atom at " + + formatv("{0:x16}", LSDAAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + + // LSDA looks good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, + 0); + } + + return Error::success(); +} + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) { + return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, + FDEToCIERelocKind, FDEToTargetRelocKind) + .atomize(); +} + +// Determine whether we can register EH tables. +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)) +#define HAVE_EHTABLE_SUPPORT 1 +#else +#define HAVE_EHTABLE_SUPPORT 0 +#endif + +#if HAVE_EHTABLE_SUPPORT +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error("could not register eh-frame: " + "__register_frame function not found"); +} + +static void deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error("could not deregister eh-frame: " + "__deregister_frame function not found"); +} +#endif + +#ifdef __APPLE__ + +template +Error walkAppleEHFrameSection(const char *const SectionStart, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + uint64_t Size = *reinterpret_cast(CurCFIRecord); + + while (Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast(OffsetField); + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + CurCFIRecord += Size; + + Size = *reinterpret_cast(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast(EHFrameSectionAddr), + registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast(EHFrameSectionAddr), + deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +AtomGraphPassFunction createEHFrameRecorderPass(const Triple &TT, + JITTargetAddress &EHFrameAddr) { + const char *EHFrameSectionName = nullptr; + if (TT.getObjectFormat() == Triple::MachO) + EHFrameSectionName = "__eh_frame"; + else + EHFrameSectionName = ".eh_frame"; + + auto RecordEHFrame = [EHFrameSectionName, + &EHFrameAddr](AtomGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first atom + // in it. + JITTargetAddress Addr = 0; + for (auto &S : G.sections()) + if (S.getName() == EHFrameSectionName && !S.atoms_empty()) { + Addr = (*S.atoms().begin())->getAddress(); + break; + } + + EHFrameAddr = Addr; + return Error::success(); + }; + + return RecordEHFrame; +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h @@ -0,0 +1,66 @@ +//===----- JITLink_EHFrameSupport.h - JITLink eh-frame utils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H + +#include "llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/BinaryStreamReader.h" + +namespace llvm { +namespace jitlink { + +/// A generic parser for eh-frame sections. +/// +/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and +/// FDEToTarget relocation kinds. +class EHFrameParser { +public: + EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind); + Error atomize(); + +private: + struct AugmentationInfo { + bool AugmentationDataPresent = false; + bool EHDataFieldPresent = false; + uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; + }; + + Expected parseAugmentationString(); + Expected readAbsolutePointer(); + Error processCIE(); + Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + + AtomGraph &G; + Section &EHFrameSection; + StringRef EHFrameContent; + JITTargetAddress EHFrameAddress; + BinaryStreamReader EHFrameReader; + DefinedAtom *CurRecordAtom = nullptr; + DefinedAtom *MostRecentCIE = nullptr; + bool LSDAFieldPresent = false; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; +}; + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp @@ -0,0 +1,73 @@ +//===------------ JITLink.cpp - Run-time JIT linker for MachO -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +void jitLink_MachO(std::unique_ptr Ctx) { + + // We don't want to do full MachO validation here. Just parse enough of the + // header to find out what MachO linker to use. + + StringRef Data = Ctx->getObjectBuffer().getBuffer(); + if (Data.size() < 4) { + Ctx->notifyFailed(make_error("Truncated MachO buffer")); + return; + } + + uint32_t Magic; + memcpy(&Magic, Data.data(), sizeof(uint32_t)); + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) + << "\n"; + }); + + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) { + Ctx->notifyFailed( + make_error("MachO 32-bit platforms not supported")); + return; + } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + MachO::mach_header_64 Header; + + memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64)); + if (Magic == MachO::MH_CIGAM_64) + swapStruct(Header); + + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: cputype = " + << format("0x%08" PRIx32, Header.cputype) + << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype) + << "\n"; + }); + + switch (Header.cputype) { + case MachO::CPU_TYPE_X86_64: + return jitLink_MachO_x86_64(std::move(Ctx)); + } + Ctx->notifyFailed(make_error("MachO-64 CPU type not valid")); + return; + } + + Ctx->notifyFailed(make_error("MachO magic not valid")); +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp @@ -0,0 +1,621 @@ +//===------- JITLink_MachO_x86_64.cpp - JIT linker functionality ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +public: + MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOAtomGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { + return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), + EHFrameSection.getContent(), + EHFrameSection.getAddress(), NegDelta32, Delta64); + }); + } + +private: + static Expected + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel && RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta'. They may + // be turned into NegDelta by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error("Unsupported x86-64 relocation kind"); + } + + Expected findAtomBySymbolIndex(const MachO::relocation_info &RI) { + auto &Obj = getObject(); + if (RI.r_symbolnum >= NumSymbols) + return make_error("Symbol index out of range"); + auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); + auto Name = SymI->getName(); + if (!Name) + return Name.takeError(); + return getGraph().getAtomByName(*Name); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = std::tuple; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected + parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error("x86_64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + auto FromAtom = findAtomBySymbolIndex(SubRI); + if (!FromAtom) + return FromAtom.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const ulittle64_t *)FixupContent; + else + FixupValue = *(const ulittle32_t *)FixupContent; + + // Find 'ToAtom' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Atom *ToAtom = nullptr; + if (UnsignedRI.r_extern) { + // Find target atom by symbol index. + if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + } else { + if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + FixupValue -= ToAtom->getAddress(); + } + + MachOX86RelocationKind DeltaKind; + Atom *TargetAtom; + uint64_t Addend; + if (&AtomToFix == &*FromAtom) { + TargetAtom = ToAtom; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + // FIXME: handle extern 'from'. + } else if (&AtomToFix == ToAtom) { + TargetAtom = &*FromAtom; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + } else { + // AtomToFix was neither FromAtom nor ToAtom. + return make_error("SUBTRACTOR relocation must fix up " + "either 'A' or 'B'"); + } + + return PairRelocInfo(DeltaKind, TargetAtom, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &G = getGraph(); + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + + // Find the atom that the fixup points to. + DefinedAtom *AtomToFix = nullptr; + { + auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); + if (!AtomToFixOrErr) + return AtomToFixOrErr.takeError(); + AtomToFix = &*AtomToFixOrErr; + } + + if (FixupAddress + (1 << RI.r_length) > + AtomToFix->getAddress() + AtomToFix->getContent().size()) + return make_error( + "Relocation content extends past end of fixup atom"); + + // Get a pointer to the fixup content. + const char *FixupContent = AtomToFix->getContent().data() + + (FixupAddress - AtomToFix->getAddress()); + + // The target atom and addend will be populated by the switch below. + Atom *TargetAtom = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = 1 << (*Kind - PCRel32Minus1Anon); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetAtom, Addend) = *PairInfo; + assert(TargetAtom && "No target atom from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Addend); + printEdge(dbgs(), *AtomToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachOInPlaceGOTAndStubsBuilder { +public: + MachOInPlaceGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + + void run() { + // We're going to be adding new atoms, but we don't want to iterate over + // the newly added ones, so just copy the existing atoms out. + std::vector DAs(G.defined_atoms().begin(), + G.defined_atoms().end()); + + for (auto *DA : DAs) + for (auto &E : DA->edges()) + if (E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) + fixGOTEdge(E); + else if (E.getKind() == Branch32 && !E.getTarget().isDefined()) + fixExternalBranchEdge(E); + } + + Atom &getGOTEntryAtom(Atom &Target) { + assert(!Target.getName().empty() && + "GOT load edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + // Build a GOT section if we don't have one already. + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ, false); + + auto &GOTEntryAtom = G.addAnonymousAtom(*GOTSection, 0x0, 8); + GOTEntryAtom.setContent( + StringRef(reinterpret_cast(NullGOTEntryContent), 8)); + GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntryAtom)) + .first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + return *GOTEntryI->second; + } + + void fixGOTEdge(Edge &E) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + auto &GOTEntryAtom = getGOTEntryAtom(E.getTarget()); + E.setKind(PCRel32); + E.setTarget(GOTEntryAtom); + // Leave the edge addend as-is. + } + + Atom &getStubAtom(Atom &Target) { + assert(!Target.getName().empty() && + "Branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + // Build a Stubs section if we don't have one already. + if (!StubsSection) { + auto StubsProt = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt, false); + } + + auto &StubAtom = G.addAnonymousAtom(*StubsSection, 0x0, 2); + StubAtom.setContent( + StringRef(reinterpret_cast(StubContent), 6)); + + // Re-use GOT entries for stub targets. + auto &GOTEntryAtom = getGOTEntryAtom(Target); + StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); + + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub atom"); + return *StubI->second; + } + + void fixExternalBranchEdge(Edge &E) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(getStubAtom(E.getTarget())); + } + + AtomGraph &G; + DenseMap GOTEntries; + DenseMap Stubs; + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachOInPlaceGOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachOInPlaceGOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker { + friend class JITLinker; + +public: + MachOJITLinker_x86_64(std::unique_ptr Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + Expected> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Target \"" << E.getTarget() << "\" out of range"; + } + return make_error(std::move(ErrMsg)); + } + + Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + using namespace support; + + char *FixupPtr = AtomWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch32: + case PCRel32: + case PCRel32Anon: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +void jitLink_MachO_x86_64(std::unique_ptr Ctx) { + PassConfiguration Config; + Triple TT("x86_64-apple-macosx"); + + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllAtomsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + MachOInPlaceGOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); +} + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/JITLink/LLVMBuild.txt =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/LLVMBuild.txt +++ llvm/trunk/lib/ExecutionEngine/JITLink/LLVMBuild.txt @@ -0,0 +1,21 @@ +;===----- ./lib/ExecutionEngine/JTILink/LLVMBuild.txt ----------*- Conf -*--===; +; +; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = JITLink +parent = ExecutionEngine +required_libraries = Object Support Index: llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h +++ llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h @@ -0,0 +1,119 @@ +//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "JITLinkGeneric.h" + +#include "llvm/Object/MachO.h" + +namespace llvm { +namespace jitlink { + +class MachOAtomGraphBuilder { +public: + virtual ~MachOAtomGraphBuilder(); + Expected> buildGraph(); + +protected: + using OffsetToAtomMap = std::map; + + class MachOSection { + public: + MachOSection() = default; + + /// Create a MachO section with the given content. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment, StringRef Content) + : Address(Address), GenericSection(&GenericSection), + ContentPtr(Content.data()), Size(Content.size()), + Alignment(Alignment) {} + + /// Create a zero-fill MachO section with the given size. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment, size_t ZeroFillSize) + : Address(Address), GenericSection(&GenericSection), Size(ZeroFillSize), + Alignment(Alignment) {} + + /// Create a section without address, content or size (used for common + /// symbol sections). + MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} + + Section &getGenericSection() const { + assert(GenericSection && "Section is null"); + return *GenericSection; + } + + StringRef getName() const { + assert(GenericSection && "No generic section attached"); + return GenericSection->getName(); + } + + bool isZeroFill() const { return !ContentPtr; } + + bool empty() const { return getSize() == 0; } + + size_t getSize() const { return Size; } + + StringRef getContent() const { + assert(ContentPtr && "getContent() called on zero-fill section"); + return {ContentPtr, Size}; + } + + JITTargetAddress getAddress() const { return Address; } + + unsigned getAlignment() const { return Alignment; } + + private: + JITTargetAddress Address = 0; + Section *GenericSection = nullptr; + const char *ContentPtr = nullptr; + size_t Size = 0; + unsigned Alignment = 0; + }; + + using CustomAtomizeFunction = std::function; + + MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); + + AtomGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); + + virtual Error addRelocations() = 0; + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + MachOSection &getCommonSection(); + + Error parseSections(); + Error addNonCustomAtoms(); + Error addAtoms(); + + const object::MachOObjectFile &Obj; + std::unique_ptr G; + DenseMap Sections; + StringMap CustomAtomizeFunctions; + Optional CommonSymbolsSection; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H Index: llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp +++ llvm/trunk/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp @@ -0,0 +1,289 @@ +//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} + +Expected> MachOAtomGraphBuilder::buildGraph() { + if (auto Err = parseSections()) + return std::move(Err); + + if (auto Err = addAtoms()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(llvm::make_unique(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, + CustomAtomizeFunction Atomizer) { + assert(!CustomAtomizeFunctions.count(SectionName) && + "Custom atomizer for this section already exists"); + CustomAtomizeFunctions[SectionName] = std::move(Atomizer); +} + +unsigned +MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { + if (!CommonSymbolsSection) { + auto Prot = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + auto &GenericSection = G->createSection("", Prot, true); + CommonSymbolsSection = MachOSection(GenericSection); + } + return *CommonSymbolsSection; +} + +Error MachOAtomGraphBuilder::parseSections() { + for (auto &SecRef : Obj.sections()) { + assert((SecRef.getAlignment() <= std::numeric_limits::max()) && + "Section alignment does not fit in 32 bits"); + + StringRef Name; + if (auto EC = SecRef.getName(Name)) + return errorCodeToError(EC); + + StringRef Content; + + // If this is a virtual section, leave its content empty. + if (!SecRef.isVirtual()) { + if (auto EC = SecRef.getContents(Content)) + return errorCodeToError(EC); + if (Content.size() != SecRef.getSize()) + return make_error("Section content size does not match " + "declared size for " + + Name); + } + + unsigned SectionIndex = SecRef.getIndex() + 1; + + LLVM_DEBUG({ + dbgs() << "Adding section " << Name << ": " + << format("0x%016" PRIx64, SecRef.getAddress()) + << ", size: " << Content.size() + << ", align: " << SecRef.getAlignment() << "\n"; + }); + + // FIXME: Get real section permissions + // How, exactly, on MachO? + sys::Memory::ProtectionFlags Prot; + if (SecRef.isText()) + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto &GenericSection = G->createSection(Name, Prot, SecRef.isBSS()); + if (SecRef.isVirtual()) + Sections[SectionIndex] = + MachOSection(GenericSection, SecRef.getAddress(), + SecRef.getAlignment(), SecRef.getSize()); + Sections[SectionIndex] = MachOSection(GenericSection, SecRef.getAddress(), + SecRef.getAlignment(), Content); + } + + return Error::success(); +} + +// Adds atoms with identified start addresses (but not lengths) for all named +// atoms. +// Also, for every section that contains named atoms, but does not have an +// atom at offset zero of that section, constructs an anonymous atom covering +// that range. +Error MachOAtomGraphBuilder::addNonCustomAtoms() { + using AddrToAtomMap = std::map; + DenseMap SecToAtoms; + + DenseMap FirstOrdinal; + + for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; + ++SymI) { + object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + + auto Addr = Sym.getAddress(); + if (!Addr) + return Addr.takeError(); + + auto SymType = Sym.getType(); + if (!SymType) + return SymType.takeError(); + + auto Flags = Sym.getFlags(); + + if (Flags & object::SymbolRef::SF_Undefined) { + LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); + G->addExternalAtom(*Name); + continue; + } else if (Flags & object::SymbolRef::SF_Absolute) { + LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " + << format("0x%016" PRIx64, *Addr) << "\n"); + auto &A = G->addAbsoluteAtom(*Name, *Addr); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + continue; + } else if (Flags & object::SymbolRef::SF_Common) { + LLVM_DEBUG({ + dbgs() << "Adding common \"" << *Name + << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; + }); + auto &A = + G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U), + Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + continue; + } + + LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); + + // This atom is neither undefined nor absolute, so it must be defined in + // this object. Get its section index. + auto SecItr = Sym.getSection(); + if (!SecItr) + return SecItr.takeError(); + + uint64_t SectionIndex = (*SecItr)->getIndex() + 1; + + LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); + + auto SecByIndexItr = Sections.find(SectionIndex); + if (SecByIndexItr == Sections.end()) + return make_error("Unrecognized section index in macho"); + + auto &Sec = SecByIndexItr->second; + + auto &A = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U)); + + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + + A.setCallable(*SymType & object::SymbolRef::ST_Function); + + LLVM_DEBUG({ + dbgs() << " Added " << *Name + << " addr: " << format("0x%016" PRIx64, *Addr) + << ", align: " << A.getAlignment() + << ", section: " << Sec.getGenericSection().getName() << "\n"; + }); + + auto &SecAtoms = SecToAtoms[&Sec]; + SecAtoms[A.getAddress() - Sec.getAddress()] = &A; + } + + // Add anonymous atoms. + for (auto &KV : Sections) { + auto &S = KV.second; + + // Skip empty sections. + if (S.empty()) + continue; + + // Skip sections with custom handling. + if (CustomAtomizeFunctions.count(S.getName())) + continue; + + auto SAI = SecToAtoms.find(&S); + + // If S is not in the SecToAtoms map then it contained no named atom. Add + // one anonymous atom to cover the whole section. + if (SAI == SecToAtoms.end()) { + SecToAtoms[&S][0] = &G->addAnonymousAtom( + S.getGenericSection(), S.getAddress(), S.getAlignment()); + continue; + } + + // Otherwise, check whether this section had an atom covering offset zero. + // If not, add one. + auto &SecAtoms = SAI->second; + if (!SecAtoms.count(0)) + SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getAlignment()); + } + + LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); + + // Set atom contents. + for (auto &KV : SecToAtoms) { + auto &S = *KV.first; + auto &SecAtoms = KV.second; + + // Iterate the atoms in reverse order and set up their contents. + JITTargetAddress LastAtomAddr = S.getSize(); + for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { + auto Offset = I->first; + auto &A = *I->second; + LLVM_DEBUG({ + dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " + << S.getAddress() + LastAtomAddr << " ]\n"; + }); + if (S.isZeroFill()) + A.setZeroFill(LastAtomAddr - Offset); + else + A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); + LastAtomAddr = Offset; + } + } + + return Error::success(); +} + +Error MachOAtomGraphBuilder::addAtoms() { + // Add all named atoms. + if (auto Err = addNonCustomAtoms()) + return Err; + + // Process special sections. + for (auto &KV : Sections) { + auto &S = KV.second; + auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); + if (HI != CustomAtomizeFunctions.end()) { + auto &Atomize = HI->second; + if (auto Err = Atomize(S)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm Index: llvm/trunk/lib/ExecutionEngine/LLVMBuild.txt =================================================================== --- llvm/trunk/lib/ExecutionEngine/LLVMBuild.txt +++ llvm/trunk/lib/ExecutionEngine/LLVMBuild.txt @@ -15,7 +15,8 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT Orc PerfJITEvents +subdirectories = Interpreter MCJIT JITLink RuntimeDyld IntelJITEvents + OProfileJIT Orc PerfJITEvents [component_0] type = Library Index: llvm/trunk/lib/ExecutionEngine/Orc/CMakeLists.txt =================================================================== --- llvm/trunk/lib/ExecutionEngine/Orc/CMakeLists.txt +++ llvm/trunk/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -11,6 +11,7 @@ Layer.cpp LLJIT.cpp NullResolver.cpp + ObjectLinkingLayer.cpp ObjectTransformLayer.cpp OrcABISupport.cpp OrcCBindings.cpp Index: llvm/trunk/lib/ExecutionEngine/Orc/Core.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/Orc/Core.cpp +++ llvm/trunk/lib/ExecutionEngine/Orc/Core.cpp @@ -26,17 +26,17 @@ #ifndef NDEBUG -cl::opt PrintHidden("debug-orc-print-hidden", cl::init(false), +cl::opt PrintHidden("debug-orc-print-hidden", cl::init(true), cl::desc("debug print hidden symbols defined by " "materialization units"), cl::Hidden); -cl::opt PrintCallable("debug-orc-print-callable", cl::init(false), +cl::opt PrintCallable("debug-orc-print-callable", cl::init(true), cl::desc("debug print callable symbols defined by " "materialization units"), cl::Hidden); -cl::opt PrintData("debug-orc-print-data", cl::init(false), +cl::opt PrintData("debug-orc-print-data", cl::init(true), cl::desc("debug print data symbols defined by " "materialization units"), cl::Hidden); @@ -1051,9 +1051,11 @@ void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, bool SearchThisJITDylibFirst, bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } @@ -1450,77 +1452,55 @@ Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; - SymbolNameSet MUDefsOverridden; - - struct ExistingDefOverriddenEntry { - SymbolMap::iterator ExistingDefItr; - JITSymbolFlags NewFlags; - }; - std::vector ExistingDefsOverridden; + std::vector ExistingDefsOverridden; + std::vector MUDefsOverridden; - for (auto &KV : MU.getSymbols()) { + for (const auto &KV : MU.getSymbols()) { assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); assert(!KV.second.isMaterializing() && "Materializing flags should be managed internally."); - SymbolMap::iterator EntryItr; - bool Added; - - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Lazy; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + auto I = Symbols.find(KV.first); - if (!Added) { + if (I != Symbols.end()) { if (KV.second.isStrong()) { - if (EntryItr->second.getFlags().isStrong() || - (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + if (I->second.getFlags().isStrong() || + I->second.getFlags().isMaterializing()) Duplicates.insert(KV.first); - else - ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + else { + assert(I->second.getFlags().isLazy() && + !I->second.getFlags().isMaterializing() && + "Overridden existing def should be in the Lazy state"); + ExistingDefsOverridden.push_back(KV.first); + } } else - MUDefsOverridden.insert(KV.first); + MUDefsOverridden.push_back(KV.first); } } - if (!Duplicates.empty()) { - // We need to remove the symbols we added. - for (auto &KV : MU.getSymbols()) { - if (Duplicates.count(KV.first)) - continue; - - bool Found = false; - for (const auto &EDO : ExistingDefsOverridden) - if (EDO.ExistingDefItr->first == KV.first) - Found = true; - - if (!Found) - Symbols.erase(KV.first); - } - - // FIXME: Return all duplicates. + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) return make_error(**Duplicates.begin()); - } - // Update flags on existing defs and call discard on their materializers. - for (auto &EDO : ExistingDefsOverridden) { - assert(EDO.ExistingDefItr->second.getFlags().isLazy() && - !EDO.ExistingDefItr->second.getFlags().isMaterializing() && - "Overridden existing def should be in the Lazy state"); + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); - EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { - auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + auto UMII = UnmaterializedInfos.find(S); assert(UMII != UnmaterializedInfos.end() && "Overridden existing def should have an UnmaterializedInfo"); - - UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + UMII->second->MU->doDiscard(*this, S); } - // Discard overridden symbols povided by MU. - for (auto &Sym : MUDefsOverridden) - MU.doDiscard(*this, Sym); + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Lazy; + Symbols[KV.first] = JITEvaluatedSymbol(0, NewFlags); + } return Error::success(); } Index: llvm/trunk/lib/ExecutionEngine/Orc/ExecutionUtils.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ llvm/trunk/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -178,18 +178,19 @@ } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), - GlobalPrefix(DL.getGlobalPrefix()) {} + GlobalPrefix(GlobalPrefix) {} Expected -DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); } SymbolNameSet DynamicLibrarySearchGenerator:: @@ -209,7 +210,8 @@ if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) continue; - std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( Index: llvm/trunk/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ llvm/trunk/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,382 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h" + +#include + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup( + SearchOrder, std::move(InternedSymbols), std::move(OnResolve), + // OnReady: + [&ES](Error Err) { ES.reportError(std::move(Err)); }, + // RegisterDependencies: + [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.resolve(InternedResult); + + if (Layer.NotifyLoaded) + Layer.NotifyLoaded(MR.getVModuleKey()); + } + + void notifyFinalized( + std::unique_ptr A) override { + + if (EHFrameAddr) { + // If there is an eh-frame then try to register it. + if (auto Err = registerEHFrameSection((void *)EHFrameAddr)) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + return; + } + } + + MR.emit(); + Layer.notifyFinalized( + ObjectLinkingLayer::ObjectResources(std::move(A), EHFrameAddr)); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Config.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, EHFrameAddr)); + + if (Layer.ModifyPassConfig) + Layer.ModifyPassConfig(TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet DAAnonDeps; + }; + std::vector Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr ObjBuffer; + DenseMap NamedSymbolDeps; + JITTargetAddress EHFrameAddr = 0; +}; + +ObjectLinkingLayer::ObjectLinkingLayer( + ExecutionSession &ES, JITLinkMemoryManager &MemMgr, + NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted, + ModifyPassConfigFunction ModifyPassConfig) + : ObjectLayer(ES), MemMgr(MemMgr), NotifyLoaded(std::move(NotifyLoaded)), + NotifyEmitted(std::move(NotifyEmitted)), + ModifyPassConfig(std::move(ModifyPassConfig)) {} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique( + *this, std::move(R), std::move(O))); +} + +ObjectLinkingLayer::ObjectResources::ObjectResources( + AllocPtr Alloc, JITTargetAddress EHFrameAddr) + : Alloc(std::move(Alloc)), EHFrameAddr(EHFrameAddr) {} + +ObjectLinkingLayer::ObjectResources::ObjectResources(ObjectResources &&Other) + : Alloc(std::move(Other.Alloc)), EHFrameAddr(Other.EHFrameAddr) { + Other.EHFrameAddr = 0; +} + +ObjectLinkingLayer::ObjectResources & +ObjectLinkingLayer::ObjectResources::operator=(ObjectResources &&Other) { + std::swap(Alloc, Other.Alloc); + std::swap(EHFrameAddr, Other.EHFrameAddr); + return *this; +} + +ObjectLinkingLayer::ObjectResources::~ObjectResources() { + const char *ErrBanner = + "ObjectLinkingLayer received error deallocating object resources:"; + + assert((EHFrameAddr == 0 || Alloc) && + "Non-null EHFrameAddr must have an associated allocation"); + + if (EHFrameAddr) + if (auto Err = deregisterEHFrameSection((void *)EHFrameAddr)) + logAllUnhandledErrors(std::move(Err), llvm::errs(), ErrBanner); + + if (Alloc) + if (auto Err = Alloc->deallocate()) + logAllUnhandledErrors(std::move(Err), llvm::errs(), ErrBanner); +} + +} // End namespace orc. +} // End namespace llvm. Index: llvm/trunk/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp =================================================================== --- llvm/trunk/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ llvm/trunk/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -47,7 +47,7 @@ // it may be found at runtime in a dynamically-loaded library. // For example, this happens when building LLVM with Visual C++ // but using the MingW runtime. -void __register_frame(void *p) { +static void __register_frame(void *p) { static bool Searched = false; static void((*rf)(void *)) = 0; @@ -60,7 +60,7 @@ rf(p); } -void __deregister_frame(void *p) { +static void __deregister_frame(void *p) { static bool Searched = false; static void((*df)(void *)) = 0; Index: llvm/trunk/lib/Support/Memory.cpp =================================================================== --- llvm/trunk/lib/Support/Memory.cpp +++ llvm/trunk/lib/Support/Memory.cpp @@ -15,6 +15,10 @@ #include "llvm/Config/llvm-config.h" #include "llvm/Support/Valgrind.h" +#ifndef NDEBUG +#include "llvm/Support/raw_ostream.h" +#endif // ifndef NDEBUG + // Include the platform-specific parts of this class. #ifdef LLVM_ON_UNIX #include "Unix/Memory.inc" @@ -22,3 +26,28 @@ #ifdef _WIN32 #include "Windows/Memory.inc" #endif + +#ifndef NDEBUG + +namespace llvm { +namespace sys { + +raw_ostream &operator<<(raw_ostream &OS, const Memory::ProtectionFlags &PF) { + assert((PF & ~(Memory::MF_READ | Memory::MF_WRITE | Memory::MF_EXEC)) == 0 && + "Unrecognized flags"); + + return OS << (PF & Memory::MF_READ ? 'R' : '-') + << (PF & Memory::MF_WRITE ? 'W' : '-') + << (PF & Memory::MF_EXEC ? 'X' : '-'); +} + +raw_ostream &operator<<(raw_ostream &OS, const MemoryBlock &MB) { + return OS << "[ " << MB.base() << " .. " + << (void *)((char *)MB.base() + MB.size()) << " ] (" << MB.size() + << " bytes)"; +} + +} // end namespace sys +} // end namespace llvm + +#endif // ifndef NDEBUG Index: llvm/trunk/test/ExecutionEngine/JITLink/MachO_x86-64_relocations.s =================================================================== --- llvm/trunk/test/ExecutionEngine/JITLink/MachO_x86-64_relocations.s +++ llvm/trunk/test/ExecutionEngine/JITLink/MachO_x86-64_relocations.s @@ -0,0 +1,203 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj -o %t/test_x86-64.o %s +# RUN: llvm-jitlink -noexec -define-abs external_data=0xdeadbeef -define-abs external_func=0xcafef00d -check=%s %t/test_x86-64.o + + .section __TEXT,__text,regular,pure_instructions + + .align 4, 0x90 +Lanon_func: + retq + + .globl named_func + .align 4, 0x90 +named_func: + xorq %rax, %rax + retq + +# Check X86_64_RELOC_BRANCH handling with a call to a local function. +# +# jitlink-check: decode_operand(test_local_call, 0) = named_func - next_pc(test_local_call) + .globl test_local_call + .align 4, 0x90 +test_local_call: + callq named_func + retq + + .globl _main + .align 4, 0x90 +_main: + retq + +# Check X86_64_RELOC_GOTPCREL handling with a load from an external symbol. +# Validate both the reference to the GOT entry, and also the content of the GOT +# entry. +# +# jitlink-check: decode_operand(test_gotld, 4) = got_addr(test_x86-64.o, external_data) - next_pc(test_gotld) +# jitlink-check: *{8}(got_addr(test_x86-64.o, external_data)) = external_data + .globl test_gotld + .align 4, 0x90 +test_gotld: + movq external_data@GOTPCREL(%rip), %rax + retq + +# Check that calls to external functions trigger the generation of stubs and GOT +# entries. +# +# jitlink-check: decode_operand(test_external_call, 0) = stub_addr(test_x86-64.o, external_func) - next_pc(test_external_call) +# jitlink-check: *{8}(got_addr(test_x86-64.o, external_func)) = external_func + .globl test_external_call + .align 4, 0x90 +test_external_call: + callq external_func + retq + +# Check signed relocation handling: +# +# X86_64_RELOC_SIGNED / Extern -- movq address of linker global +# X86_64_RELOC_SIGNED1 / Extern -- movb immediate byte to linker global +# X86_64_RELOC_SIGNED2 / Extern -- movw immediate word to linker global +# X86_64_RELOC_SIGNED4 / Extern -- movl immediate long to linker global +# +# X86_64_RELOC_SIGNED / Anon -- movq address of linker private into register +# X86_64_RELOC_SIGNED1 / Anon -- movb immediate byte to linker private +# X86_64_RELOC_SIGNED2 / Anon -- movw immediate word to linker private +# X86_64_RELOC_SIGNED4 / Anon -- movl immediate long to linker private +signed_reloc_checks: + .globl signed +# jitlink-check: decode_operand(signed, 4) = named_data - next_pc(signed) +signed: + movq named_data(%rip), %rax + + .globl signed1 +# jitlink-check: decode_operand(signed1, 3) = named_data - next_pc(signed1) +signed1: + movb $0xAA, named_data(%rip) + + .globl signed2 +# jitlink-check: decode_operand(signed2, 3) = named_data - next_pc(signed2) +signed2: + movw $0xAAAA, named_data(%rip) + + .globl signed4 +# jitlink-check: decode_operand(signed4, 3) = named_data - next_pc(signed4) +signed4: + movl $0xAAAAAAAA, named_data(%rip) + + .globl signedanon +# jitlink-check: decode_operand(signedanon, 4) = section_addr(test_x86-64.o, __data) - next_pc(signedanon) +signedanon: + movq Lanon_data(%rip), %rax + + .globl signed1anon +# jitlink-check: decode_operand(signed1anon, 3) = section_addr(test_x86-64.o, __data) - next_pc(signed1anon) +signed1anon: + movb $0xAA, Lanon_data(%rip) + + .globl signed2anon +# jitlink-check: decode_operand(signed2anon, 3) = section_addr(test_x86-64.o, __data) - next_pc(signed2anon) +signed2anon: + movw $0xAAAA, Lanon_data(%rip) + + .globl signed4anon +# jitlink-check: decode_operand(signed4anon, 3) = section_addr(test_x86-64.o, __data) - next_pc(signed4anon) +signed4anon: + movl $0xAAAAAAAA, Lanon_data(%rip) + + + + .section __DATA,__data + +# Storage target for non-extern X86_64_RELOC_SIGNED_(1/2/4) relocs. + .p2align 3 +Lanon_data: + .quad 0x1111111111111111 + +# Check X86_64_RELOC_SUBTRACTOR Quad/Long in anonymous storage with anonymous minuend +# Only the form "LA: .quad LA - B + C" is tested. The form "LA: .quad B - LA + C" is +# invalid because the minuend can not be local. +# +# Note: +8 offset in expression below to accounts for sizeof(Lanon_data). +# jitlink-check: *{8}(section_addr(test_x86-64.o, __data) + 8) = (section_addr(test_x86-64.o, __data) + 8) - named_data + 2 + .p2align 3 +Lanon_minuend_quad: + .quad Lanon_minuend_quad - named_data + 2 + +# Note: +16 offset in expression below to accounts for sizeof(Lanon_data) + sizeof(Lanon_minuend_long). +# jitlink-check: *{4}(section_addr(test_x86-64.o, __data) + 16) = ((section_addr(test_x86-64.o, __data) + 16) - named_data + 2)[31:0] + .p2align 2 +Lanon_minuend_long: + .long Lanon_minuend_long - named_data + 2 + + +# Named quad storage target (first named atom in __data). + .globl named_data + .p2align 3 +named_data: + .quad 0x2222222222222222 + +# Check X86_64_RELOC_UNSIGNED / extern handling by putting the address of a +# local named function in a pointer variable. +# +# jitlink-check: *{8}named_func_addr = named_func + .globl named_func_addr + .p2align 3 +named_func_addr: + .quad named_func + +# Check X86_64_RELOC_UNSIGNED / non-extern handling by putting the address of a +# local anonymous function in a pointer variable. +# +# jitlink-check: *{8}anon_func_addr = section_addr(test_x86-64.o, __text) + .globl anon_func_addr + .p2align 3 +anon_func_addr: + .quad Lanon_func + +# X86_64_RELOC_SUBTRACTOR Quad/Long in named storage with anonymous minuend +# +# jitlink-check: *{8}minuend_quad1 = section_addr(test_x86-64.o, __data) - minuend_quad1 + 2 +# Only the form "B: .quad LA - B + C" is tested. The form "B: .quad B - LA + C" is +# invalid because the minuend can not be local. + .globl minuend_quad1 + .p2align 3 +minuend_quad1: + .quad Lanon_data - minuend_quad1 + 2 + +# jitlink-check: *{4}minuend_long1 = (section_addr(test_x86-64.o, __data) - minuend_long1 + 2)[31:0] + .globl minuend_long1 + .p2align 2 +minuend_long1: + .long Lanon_data - minuend_long1 + 2 + +# Check X86_64_RELOC_SUBTRACTOR Quad/Long in named storage with minuend and subtrahend. +# Both forms "A: .quad A - B + C" and "A: .quad B - A + C" are tested. +# +# Check "A: .quad B - A + C". +# jitlink-check: *{8}minuend_quad2 = (named_data - minuend_quad2 + 2) + .globl minuend_quad2 + .p2align 3 +minuend_quad2: + .quad named_data - minuend_quad2 + 2 + +# Check "A: .long B - A + C". +# jitlink-check: *{4}minuend_long2 = (named_data - minuend_long2 + 2)[31:0] + .globl minuend_long2 + .p2align 2 +minuend_long2: + .long named_data - minuend_long2 + 2 + +# Check "A: .quad A - B + C". +# jitlink-check: *{8}minuend_quad3 = (minuend_quad3 - named_data + 2) + .globl minuend_quad3 + .p2align 3 +minuend_quad3: + .quad minuend_quad3 - named_data + 2 + +# Check "A: .long B - A + C". +# jitlink-check: *{4}minuend_long3 = (minuend_long3 - named_data + 2)[31:0] + .globl minuend_long3 + .p2align 2 +minuend_long3: + .long minuend_long3 - named_data + 2 + +.subsections_via_symbols Index: llvm/trunk/tools/lli/lli.cpp =================================================================== --- llvm/trunk/tools/lli/lli.cpp +++ llvm/trunk/tools/lli/lli.cpp @@ -800,7 +800,8 @@ return Dump(std::move(TSM), R); }); J->getMainJITDylib().setGenerator( - ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(DL))); + ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( + DL.getGlobalPrefix()))); orc::MangleAndInterner Mangle(J->getExecutionSession(), DL); orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; Index: llvm/trunk/tools/llvm-jitlink/CMakeLists.txt =================================================================== --- llvm/trunk/tools/llvm-jitlink/CMakeLists.txt +++ llvm/trunk/tools/llvm-jitlink/CMakeLists.txt @@ -0,0 +1,17 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + ExecutionEngine + JITLink + MC + Object + OrcJIT + RuntimeDyld + Support + ) + +add_llvm_tool(llvm-jitlink + llvm-jitlink.cpp + llvm-jitlink-macho.cpp + ) + +export_executable_symbols(llvm-jitlink) Index: llvm/trunk/tools/llvm-jitlink/LLVMBuild.txt =================================================================== --- llvm/trunk/tools/llvm-jitlink/LLVMBuild.txt +++ llvm/trunk/tools/llvm-jitlink/LLVMBuild.txt @@ -0,0 +1,21 @@ +;===- ./tools/llvm-jitlink/LLVMBuild.txt -----------------------*- Conf -*--===; +; +; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Tool +name = llvm-jitlink +parent = Tools +required_libraries = JITLink MC Object RuntimeDyld Support all-targets Index: llvm/trunk/tools/llvm-jitlink/llvm-jitlink-macho.cpp =================================================================== --- llvm/trunk/tools/llvm-jitlink/llvm-jitlink-macho.cpp +++ llvm/trunk/tools/llvm-jitlink/llvm-jitlink-macho.cpp @@ -0,0 +1,142 @@ +//===-- llvm-jitlink-macho.cpp -- MachO parsing support for llvm-jitlink --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// . +// +//===----------------------------------------------------------------------===// + +#include "llvm-jitlink.h" + +#include "llvm/Support/Error.h" +#include "llvm/Support/Path.h" + +#define DEBUG_TYPE "llvm-jitlink" + +using namespace llvm; +using namespace llvm::jitlink; + +static bool isMachOGOTSection(Section &S) { return S.getName() == "$__GOT"; } + +static bool isMachOStubsSection(Section &S) { + return S.getName() == "$__STUBS"; +} + +static Expected getFirstRelocationEdge(AtomGraph &G, DefinedAtom &DA) { + auto EItr = std::find_if(DA.edges().begin(), DA.edges().end(), + [](Edge &E) { return E.isRelocation(); }); + if (EItr == DA.edges().end()) + return make_error("GOT entry in " + G.getName() + ", \"" + + DA.getSection().getName() + + "\" has no relocations", + inconvertibleErrorCode()); + return *EItr; +} + +static Expected getMachOGOTTarget(AtomGraph &G, DefinedAtom &DA) { + auto E = getFirstRelocationEdge(G, DA); + if (!E) + return E.takeError(); + auto &TA = E->getTarget(); + if (!TA.hasName()) + return make_error("GOT entry in " + G.getName() + ", \"" + + DA.getSection().getName() + + "\" points to anonymous " + "atom", + inconvertibleErrorCode()); + if (TA.isDefined() || TA.isAbsolute()) + return make_error( + "GOT entry \"" + TA.getName() + "\" in " + G.getName() + ", \"" + + DA.getSection().getName() + "\" does not point to an external atom", + inconvertibleErrorCode()); + return TA; +} + +static Expected getMachOStubTarget(AtomGraph &G, DefinedAtom &DA) { + auto E = getFirstRelocationEdge(G, DA); + if (!E) + return E.takeError(); + auto &GOTA = E->getTarget(); + if (!GOTA.isDefined() || + !isMachOGOTSection(static_cast(GOTA).getSection())) + return make_error("Stubs entry in " + G.getName() + ", \"" + + DA.getSection().getName() + + "\" does not point to GOT entry", + inconvertibleErrorCode()); + return getMachOGOTTarget(G, static_cast(GOTA)); +} + +namespace llvm { + +Error registerMachOStubsAndGOT(Session &S, AtomGraph &G) { + auto FileName = sys::path::filename(G.getName()); + if (S.FileInfos.count(FileName)) { + return make_error("When -check is passed, file names must be " + "distinct (duplicate: \"" + + FileName + "\")", + inconvertibleErrorCode()); + } + + auto &FileInfo = S.FileInfos[FileName]; + LLVM_DEBUG({ + dbgs() << "Registering MachO file info for \"" << FileName << "\"\n"; + }); + for (auto &Sec : G.sections()) { + LLVM_DEBUG({ + dbgs() << " Section \"" << Sec.getName() << "\": " + << (Sec.atoms_empty() ? "empty. skipping." : "processing...") + << "\n"; + }); + + // Skip empty sections. + if (Sec.atoms_empty()) + continue; + + if (FileInfo.SectionInfos.count(Sec.getName())) + return make_error("Encountered duplicate section name \"" + + Sec.getName() + "\" in \"" + FileName + + "\"", + inconvertibleErrorCode()); + + bool isGOTSection = isMachOGOTSection(Sec); + bool isStubsSection = isMachOStubsSection(Sec); + + auto &SectionInfo = FileInfo.SectionInfos[Sec.getName()]; + + auto *FirstAtom = *Sec.atoms().begin(); + auto *LastAtom = FirstAtom; + for (auto *DA : Sec.atoms()) { + if (DA->getAddress() < FirstAtom->getAddress()) + FirstAtom = DA; + if (DA->getAddress() > LastAtom->getAddress()) + LastAtom = DA; + if (isGOTSection) { + if (auto TA = getMachOGOTTarget(G, *DA)) { + FileInfo.GOTEntryInfos[TA->getName()] = {DA->getContent(), + DA->getAddress()}; + } else + return TA.takeError(); + } else if (isStubsSection) { + if (auto TA = getMachOStubTarget(G, *DA)) + FileInfo.StubInfos[TA->getName()] = {DA->getContent(), + DA->getAddress()}; + else + return TA.takeError(); + } else if (DA->hasName() && DA->isGlobal()) + S.SymbolInfos[DA->getName()] = {DA->getContent(), DA->getAddress()}; + } + const char *StartAddr = FirstAtom->getContent().data(); + const char *EndAddr = + LastAtom->getContent().data() + LastAtom->getContent().size(); + SectionInfo.TargetAddress = FirstAtom->getAddress(); + SectionInfo.Content = StringRef(StartAddr, EndAddr - StartAddr); + } + + return Error::success(); +} + +} // end namespace llvm Index: llvm/trunk/tools/llvm-jitlink/llvm-jitlink.h =================================================================== --- llvm/trunk/tools/llvm-jitlink/llvm-jitlink.h +++ llvm/trunk/tools/llvm-jitlink/llvm-jitlink.h @@ -0,0 +1,72 @@ +//===---- llvm-jitlink.h - Session and format-specific decls ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for remote-JITing with LLI. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_JITLINK_LLVM_JITLINK_H +#define LLVM_TOOLS_LLVM_JITLINK_LLVM_JITLINK_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/RuntimeDyldChecker.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" + +#include + +namespace llvm { + +struct Session { + orc::ExecutionSession ES; + jitlink::InProcessMemoryManager MemMgr; + orc::ObjectLinkingLayer ObjLayer; + std::vector JDSearchOrder; + Triple TT; + + Session(Triple TT); + void dumpSessionInfo(raw_ostream &OS); + void modifyPassConfig(const Triple &FTT, + jitlink::PassConfiguration &PassConfig); + + using MemoryRegionInfo = RuntimeDyldChecker::MemoryRegionInfo; + + struct FileInfo { + StringMap SectionInfos; + StringMap StubInfos; + StringMap GOTEntryInfos; + }; + + using SymbolInfoMap = StringMap; + using FileInfoMap = StringMap; + + Expected findFileInfo(StringRef FileName); + Expected findSectionInfo(StringRef FileName, + StringRef SectionName); + Expected findStubInfo(StringRef FileName, + StringRef TargetName); + Expected findGOTEntryInfo(StringRef FileName, + StringRef TargetName); + + bool isSymbolRegistered(StringRef Name); + Expected findSymbolInfo(StringRef SymbolName, + Twine ErrorMsgStem); + + SymbolInfoMap SymbolInfos; + FileInfoMap FileInfos; + uint64_t SizeBeforePruning = 0; + uint64_t SizeAfterFixups = 0; +}; + +Error registerMachOStubsAndGOT(Session &S, jitlink::AtomGraph &G); + +} // end namespace llvm + +#endif // LLVM_TOOLS_LLVM_JITLINK_LLVM_JITLINK_H Index: llvm/trunk/tools/llvm-jitlink/llvm-jitlink.cpp =================================================================== --- llvm/trunk/tools/llvm-jitlink/llvm-jitlink.cpp +++ llvm/trunk/tools/llvm-jitlink/llvm-jitlink.cpp @@ -0,0 +1,529 @@ +//===- llvm-jitlink.cpp -- Command line interface/tester for llvm-jitlink -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This utility provides a simple command line interface to the llvm jitlink +// library, which makes relocatable object files executable in memory. Its +// primary function is as a testing utility for the jitlink library. +// +//===----------------------------------------------------------------------===// + +#include "llvm-jitlink.h" + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" + +#include +#include + +#include "dlfcn.h" + +#define DEBUG_TYPE "llvm-jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +static cl::list InputFiles(cl::Positional, cl::OneOrMore, + cl::desc("input files")); + +static cl::opt NoExec("noexec", cl::desc("Do not execute loaded code"), + cl::init(false)); + +static cl::list + CheckFiles("check", cl::desc("File containing verifier checks"), + cl::ZeroOrMore); + +static cl::opt + EntryPointName("entry", cl::desc("Symbol to call as main entry point"), + cl::init("")); + +static cl::list JITLinkDylibs( + "jld", cl::desc("Specifies the JITDylib to be used for any subsequent " + "input file arguments")); + +static cl::list + Dylibs("dlopen", cl::desc("Dynamic libraries to load before linking"), + cl::ZeroOrMore); + +static cl::opt + NoProcessSymbols("no-process-syms", + cl::desc("Do not resolve to llvm-jitlink process symbols"), + cl::init(false)); + +static cl::list +AbsoluteDefs("define-abs", + cl::desc("Inject absolute symbol definitions (syntax: =)"), + cl::ZeroOrMore); + +static cl::opt ShowAddrs( + "show-addrs", + cl::desc("Print registered symbol, section, got and stub addresses"), + cl::init(false)); + +static cl::opt ShowAtomGraph( + "show-graph", + cl::desc("Print the atom graph after fixups have been applied"), + cl::init(false)); + +static cl::opt ShowSizes( + "show-sizes", + cl::desc("Show sizes: pre- and post-dead stripping, and allocations"), + cl::init(false)); + +ExitOnError ExitOnErr; + +namespace llvm { + +static raw_ostream & +operator<<(raw_ostream &OS, const Session::MemoryRegionInfo &MRI) { + return OS << "target addr = " << format("0x%016" PRIx64, MRI.TargetAddress) + << ", content: " << (const void *)MRI.Content.data() << " -- " + << (const void *)(MRI.Content.data() + MRI.Content.size()) << " (" + << MRI.Content.size() << " bytes)"; +} + +static raw_ostream & +operator<<(raw_ostream &OS, const Session::SymbolInfoMap &SIM) { + OS << "Symbols:\n"; + for (auto &SKV : SIM) + OS << " \"" << SKV.first() << "\" " << SKV.second << "\n"; + return OS; +} + +static raw_ostream & +operator<<(raw_ostream &OS, const Session::FileInfo &FI) { + for (auto &SIKV : FI.SectionInfos) + OS << " Section \"" << SIKV.first() << "\": " << SIKV.second << "\n"; + for (auto &GOTKV : FI.GOTEntryInfos) + OS << " GOT \"" << GOTKV.first() << "\": " << GOTKV.second << "\n"; + for (auto &StubKV : FI.StubInfos) + OS << " Stub \"" << StubKV.first() << "\": " << StubKV.second << "\n"; + return OS; +} + +static raw_ostream & +operator<<(raw_ostream &OS, const Session::FileInfoMap &FIM) { + for (auto &FIKV : FIM) + OS << "File \"" << FIKV.first() << "\":\n" << FIKV.second; + return OS; +} + +static uint64_t computeTotalAtomSizes(AtomGraph &G) { + uint64_t TotalSize = 0; + for (auto *DA : G.defined_atoms()) + if (DA->isZeroFill()) + TotalSize += DA->getZeroFillSize(); + else + TotalSize += DA->getContent().size(); + return TotalSize; +} + +Session::Session(Triple TT) + : ObjLayer(ES, MemMgr, ObjectLinkingLayer::NotifyLoadedFunction(), + ObjectLinkingLayer::NotifyEmittedFunction(), + [this](const Triple &TT, PassConfiguration &PassConfig) { + modifyPassConfig(TT, PassConfig); + }), + TT(std::move(TT)) {} + +void Session::dumpSessionInfo(raw_ostream &OS) { + OS << "Registered addresses:\n" << SymbolInfos << FileInfos; +} + +void Session::modifyPassConfig(const Triple &FTT, + PassConfiguration &PassConfig) { + if (!CheckFiles.empty()) + PassConfig.PostFixupPasses.push_back([this](AtomGraph &G) { + if (TT.getObjectFormat() == Triple::MachO) + return registerMachOStubsAndGOT(*this, G); + return make_error("Unsupported object format for GOT/stub " + "registration", + inconvertibleErrorCode()); + }); + + if (ShowAtomGraph) + PassConfig.PostFixupPasses.push_back([](AtomGraph &G) -> Error { + outs() << "Atom graph post-fixup:\n"; + G.dump(outs()); + return Error::success(); + }); + + + if (ShowSizes) { + PassConfig.PrePrunePasses.push_back([this](AtomGraph &G) -> Error { + SizeBeforePruning += computeTotalAtomSizes(G); + return Error::success(); + }); + PassConfig.PostFixupPasses.push_back([this](AtomGraph &G) -> Error { + SizeAfterFixups += computeTotalAtomSizes(G); + return Error::success(); + }); + } + +} + +Expected Session::findFileInfo(StringRef FileName) { + auto FileInfoItr = FileInfos.find(FileName); + if (FileInfoItr == FileInfos.end()) + return make_error("file \"" + FileName + "\" not recognized", + inconvertibleErrorCode()); + return FileInfoItr->second; +} + +Expected +Session::findSectionInfo(StringRef FileName, StringRef SectionName) { + auto FI = findFileInfo(FileName); + if (!FI) + return FI.takeError(); + auto SecInfoItr = FI->SectionInfos.find(SectionName); + if (SecInfoItr == FI->SectionInfos.end()) + return make_error("no section \"" + SectionName + + "\" registered for file \"" + FileName + + "\"", + inconvertibleErrorCode()); + return SecInfoItr->second; +} + +Expected +Session::findStubInfo(StringRef FileName, StringRef TargetName) { + auto FI = findFileInfo(FileName); + if (!FI) + return FI.takeError(); + auto StubInfoItr = FI->StubInfos.find(TargetName); + if (StubInfoItr == FI->StubInfos.end()) + return make_error("no stub for \"" + TargetName + + "\" registered for file \"" + FileName + + "\"", + inconvertibleErrorCode()); + return StubInfoItr->second; +} + +Expected +Session::findGOTEntryInfo(StringRef FileName, StringRef TargetName) { + auto FI = findFileInfo(FileName); + if (!FI) + return FI.takeError(); + auto GOTInfoItr = FI->GOTEntryInfos.find(TargetName); + if (GOTInfoItr == FI->GOTEntryInfos.end()) + return make_error("no GOT entry for \"" + TargetName + + "\" registered for file \"" + FileName + + "\"", + inconvertibleErrorCode()); + return GOTInfoItr->second; +} + +bool Session::isSymbolRegistered(StringRef SymbolName) { + return SymbolInfos.count(SymbolName); +} + +Expected +Session::findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem) { + auto SymInfoItr = SymbolInfos.find(SymbolName); + if (SymInfoItr == SymbolInfos.end()) + return make_error(ErrorMsgStem + ": symbol " + SymbolName + + " not found", + inconvertibleErrorCode()); + return SymInfoItr->second; +} + +} // end namespace llvm + +Error loadProcessSymbols(Session &S) { + std::string ErrMsg; + if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, &ErrMsg)) + return make_error(std::move(ErrMsg), inconvertibleErrorCode()); + + char GlobalPrefix = S.TT.getObjectFormat() == Triple::MachO ? '_' : '\0'; + S.ES.getMainJITDylib().setGenerator(ExitOnErr( + orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(GlobalPrefix))); + + return Error::success(); +} + +Error loadDylibs() { + // FIXME: This should all be handled inside DynamicLibrary. + for (const auto &Dylib : Dylibs) { + if (!sys::fs::is_regular_file(Dylib)) + return make_error("\"" + Dylib + "\" is not a regular file", + inconvertibleErrorCode()); + std::string ErrMsg; + if (sys::DynamicLibrary::LoadLibraryPermanently(Dylib.c_str(), &ErrMsg)) + return make_error(ErrMsg, inconvertibleErrorCode()); + } + + return Error::success(); +} + +Triple getFirstFileTriple() { + assert(!InputFiles.empty() && "InputFiles can not be empty"); + auto ObjBuffer = + ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFiles.front()))); + auto Obj = ExitOnErr( + object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())); + return Obj->makeTriple(); +} + +Error loadObjects(Session &S) { + + std::map IdxToJLD; + + // First, set up JITDylibs. + LLVM_DEBUG(dbgs() << "Creating JITDylibs...\n"); + { + // Create a "main" JITLinkDylib. + auto &MainJD = S.ES.getMainJITDylib(); + IdxToJLD[0] = &MainJD; + S.JDSearchOrder.push_back(&MainJD); + LLVM_DEBUG(dbgs() << " 0: " << MainJD.getName() << "\n"); + + // Add any extra JITLinkDylibs from the command line. + std::string JDNamePrefix("lib"); + for (auto JLDItr = JITLinkDylibs.begin(), JLDEnd = JITLinkDylibs.end(); + JLDItr != JLDEnd; ++JLDItr) { + auto &JD = S.ES.createJITDylib(JDNamePrefix + *JLDItr); + unsigned JDIdx = + JITLinkDylibs.getPosition(JLDItr - JITLinkDylibs.begin()); + IdxToJLD[JDIdx] = &JD; + S.JDSearchOrder.push_back(&JD); + LLVM_DEBUG(dbgs() << " " << JDIdx << ": " << JD.getName() << "\n"); + } + + // Set every dylib to link against every other, in command line order. + for (auto *JD : S.JDSearchOrder) { + JITDylibSearchList O; + for (auto *JD2 : S.JDSearchOrder) { + if (JD2 == JD) + continue; + O.push_back(std::make_pair(JD2, false)); + } + JD->setSearchOrder(std::move(O)); + } + } + + // Load each object into the corresponding JITDylib.. + LLVM_DEBUG(dbgs() << "Adding objects...\n"); + for (auto InputFileItr = InputFiles.begin(), InputFileEnd = InputFiles.end(); + InputFileItr != InputFileEnd; ++InputFileItr) { + unsigned InputFileArgIdx = + InputFiles.getPosition(InputFileItr - InputFiles.begin()); + StringRef InputFile = *InputFileItr; + auto &JD = *std::prev(IdxToJLD.lower_bound(InputFileArgIdx))->second; + LLVM_DEBUG(dbgs() << " " << InputFileArgIdx << ": \"" << InputFile + << "\" to " << JD.getName() << "\n";); + auto ObjBuffer = + ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile))); + ExitOnErr(S.ObjLayer.add(JD, std::move(ObjBuffer))); + } + + // Define absolute symbols. + LLVM_DEBUG(dbgs() << "Defining absolute symbols...\n"); + for (auto AbsDefItr = AbsoluteDefs.begin(), AbsDefEnd = AbsoluteDefs.end(); + AbsDefItr != AbsDefEnd; ++AbsDefItr) { + unsigned AbsDefArgIdx = + AbsoluteDefs.getPosition(AbsDefItr - AbsoluteDefs.begin()); + auto &JD = *std::prev(IdxToJLD.lower_bound(AbsDefArgIdx))->second; + + StringRef AbsDefStmt = *AbsDefItr; + size_t EqIdx = AbsDefStmt.find_first_of('='); + if (EqIdx == StringRef::npos) + return make_error("Invalid absolute define \"" + AbsDefStmt + + "\". Syntax: =", + inconvertibleErrorCode()); + StringRef Name = AbsDefStmt.substr(0, EqIdx).trim(); + StringRef AddrStr = AbsDefStmt.substr(EqIdx + 1).trim(); + + uint64_t Addr; + if (AddrStr.getAsInteger(0, Addr)) + return make_error("Invalid address expression \"" + AddrStr + + "\" in absolute define \"" + AbsDefStmt + + "\"", + inconvertibleErrorCode()); + JITEvaluatedSymbol AbsDef(Addr, JITSymbolFlags::Exported); + if (auto Err = JD.define(absoluteSymbols({{S.ES.intern(Name), AbsDef}}))) + return Err; + + // Register the absolute symbol with the session symbol infos. + S.SymbolInfos[Name] = { StringRef(), Addr }; + } + + LLVM_DEBUG({ + dbgs() << "Dylib search order is [ "; + for (auto *JD : S.JDSearchOrder) + dbgs() << JD->getName() << " "; + dbgs() << "]\n"; + }); + + return Error::success(); +} + +Error runChecks(Session &S) { + + auto TripleName = S.TT.str(); + std::string ErrorStr; + const Target *TheTarget = TargetRegistry::lookupTarget("", S.TT, ErrorStr); + if (!TheTarget) + ExitOnErr(make_error("Error accessing target '" + TripleName + + "': " + ErrorStr, + inconvertibleErrorCode())); + + std::unique_ptr STI( + TheTarget->createMCSubtargetInfo(TripleName, "", "")); + if (!STI) + ExitOnErr( + make_error("Unable to create subtarget for " + TripleName, + inconvertibleErrorCode())); + + std::unique_ptr MRI(TheTarget->createMCRegInfo(TripleName)); + if (!MRI) + ExitOnErr(make_error("Unable to create target register info " + "for " + + TripleName, + inconvertibleErrorCode())); + + std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, TripleName)); + if (!MAI) + ExitOnErr(make_error("Unable to create target asm info " + + TripleName, + inconvertibleErrorCode())); + + MCContext Ctx(MAI.get(), MRI.get(), nullptr); + + std::unique_ptr Disassembler( + TheTarget->createMCDisassembler(*STI, Ctx)); + if (!Disassembler) + ExitOnErr(make_error("Unable to create disassembler for " + + TripleName, + inconvertibleErrorCode())); + + std::unique_ptr MII(TheTarget->createMCInstrInfo()); + + std::unique_ptr InstPrinter( + TheTarget->createMCInstPrinter(Triple(TripleName), 0, *MAI, *MII, *MRI)); + + auto IsSymbolValid = [&S](StringRef Symbol) { + return S.isSymbolRegistered(Symbol); + }; + + auto GetSymbolInfo = [&S](StringRef Symbol) { + return S.findSymbolInfo(Symbol, "Can not get symbol info"); + }; + + auto GetSectionInfo = [&S](StringRef FileName, StringRef SectionName) { + return S.findSectionInfo(FileName, SectionName); + }; + + auto GetStubInfo = [&S](StringRef FileName, StringRef SectionName) { + return S.findStubInfo(FileName, SectionName); + }; + + auto GetGOTInfo = [&S](StringRef FileName, StringRef SectionName) { + return S.findGOTEntryInfo(FileName, SectionName); + }; + + RuntimeDyldChecker Checker( + IsSymbolValid, GetSymbolInfo, GetSectionInfo, GetStubInfo, GetGOTInfo, + S.TT.isLittleEndian() ? support::little : support::big, + Disassembler.get(), InstPrinter.get(), dbgs()); + + for (auto &CheckFile : CheckFiles) { + auto CheckerFileBuf = + ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(CheckFile))); + if (!Checker.checkAllRulesInBuffer("# jitlink-check:", &*CheckerFileBuf)) + ExitOnErr(make_error( + "Some checks in " + CheckFile + " failed", inconvertibleErrorCode())); + } + + return Error::success(); +} + +static void dumpSessionStats(Session &S) { + if (ShowSizes) + outs() << "Total size of all atoms before pruning: " << S.SizeBeforePruning + << "\nTotal size of all atoms after fixups: " << S.SizeAfterFixups + << "\n"; +} + +static Expected getMainEntryPoint(Session &S) { + + // First, if the entry point has not been set, set it to a sensible default + // for this process. + if (EntryPointName.empty()) { + if (S.TT.getObjectFormat() == Triple::MachO) + EntryPointName = "_main"; + else + EntryPointName = "main"; + } + + return S.ES.lookup(S.JDSearchOrder, EntryPointName); +} + +Expected runEntryPoint(Session &S, JITEvaluatedSymbol EntryPoint) { + assert(EntryPoint.getAddress() && "Entry point address should not be null"); + + constexpr const char *JITProgramName = ""; + auto PNStorage = llvm::make_unique(strlen(JITProgramName) + 1); + strcpy(PNStorage.get(), JITProgramName); + + std::vector EntryPointArgs; + EntryPointArgs.push_back(PNStorage.get()); + EntryPointArgs.push_back(nullptr); + + using MainTy = int (*)(int, const char *[]); + MainTy EntryPointPtr = reinterpret_cast(EntryPoint.getAddress()); + + return EntryPointPtr(EntryPointArgs.size() - 1, EntryPointArgs.data()); +} + +int main(int argc, char *argv[]) { + InitLLVM X(argc, argv); + + InitializeAllTargetInfos(); + InitializeAllTargetMCs(); + InitializeAllDisassemblers(); + + cl::ParseCommandLineOptions(argc, argv, "llvm jitlink tool"); + ExitOnErr.setBanner(std::string(argv[0]) + ": "); + + Session S(getFirstFileTriple()); + + if (!NoProcessSymbols) + ExitOnErr(loadProcessSymbols(S)); + ExitOnErr(loadDylibs()); + + ExitOnErr(loadObjects(S)); + + auto EntryPoint = ExitOnErr(getMainEntryPoint(S)); + + if (ShowAddrs) + S.dumpSessionInfo(outs()); + + ExitOnErr(runChecks(S)); + + dumpSessionStats(S); + + if (NoExec) + return 0; + + return ExitOnErr(runEntryPoint(S, EntryPoint)); +} Index: llvm/trunk/unittests/ExecutionEngine/CMakeLists.txt =================================================================== --- llvm/trunk/unittests/ExecutionEngine/CMakeLists.txt +++ llvm/trunk/unittests/ExecutionEngine/CMakeLists.txt @@ -12,6 +12,7 @@ ExecutionEngineTest.cpp ) +add_subdirectory(JITLink) add_subdirectory(Orc) # Include MCJIT tests only if native arch is a built JIT target. Index: llvm/trunk/unittests/ExecutionEngine/JITLink/CMakeLists.txt =================================================================== --- llvm/trunk/unittests/ExecutionEngine/JITLink/CMakeLists.txt +++ llvm/trunk/unittests/ExecutionEngine/JITLink/CMakeLists.txt @@ -0,0 +1,15 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + JITLink + MC + Object + RuntimeDyld + Support + ) + +add_llvm_unittest(JITLinkTests + JITLinkTestCommon.cpp + JITLinkTest_MachO_x86_64_Tests.cpp + ) + +target_link_libraries(JITLinkTests PRIVATE LLVMTestingSupport) Index: llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h =================================================================== --- llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h +++ llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h @@ -0,0 +1,196 @@ +//===---- JITLinkTestCommon.h - Utilities for Orc Unit Tests ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Common utilities for JITLink unit tests. +// +//===----------------------------------------------------------------------===// + + +#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTCOMMON_H +#define LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTCOMMON_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" + +#include "gtest/gtest.h" + +namespace llvm { + +class JITLinkTestCommon { +public: + + class TestResources { + public: + TestResources(StringRef AsmSrc, StringRef TripleStr, bool PIC, + bool LargeCodeModel, MCTargetOptions Options); + + MemoryBufferRef getTestObjectBufferRef() const; + + const MCDisassembler &getDisassembler() const { return *Dis; } + + private: + void initializeTripleSpecifics(Triple &TT); + void initializeTestSpecifics(StringRef AsmSource, const Triple &TT, + bool PIC, bool LargeCodeModel); + + const Target *TheTarget = nullptr; + SourceMgr SrcMgr; + SmallVector ObjBuffer; + raw_svector_ostream ObjStream; + + MCTargetOptions Options; + std::unique_ptr MRI; + std::unique_ptr MAI; + std::unique_ptr MCII; + std::unique_ptr STI; + + MCObjectFileInfo MOFI; + std::unique_ptr AsCtx; + std::unique_ptr MOS; + + std::unique_ptr DisCtx; + std::unique_ptr Dis; + }; + + class TestJITLinkContext : public jitlink::JITLinkContext { + public: + using TestCaseFunction = std::function; + + using NotifyResolvedFunction = std::function; + + using NotifyFinalizedFunction = std::function)>; + + TestJITLinkContext(TestResources &TR, TestCaseFunction TestCase); + + StringMap &externals() { return Externals; } + + TestJITLinkContext & + setNotifyResolved(NotifyResolvedFunction NotifyResolved); + + TestJITLinkContext & + setNotifyFinalized(NotifyFinalizedFunction NotifyFinalized); + + TestJITLinkContext & + setMemoryManager(std::unique_ptr MM); + + jitlink::JITLinkMemoryManager &getMemoryManager() override; + + MemoryBufferRef getObjectBuffer() const override; + + void notifyFailed(Error Err) override; + + void + lookup(const DenseSet &Symbols, + jitlink::JITLinkAsyncLookupContinuation LookupContinuation) override; + + void notifyResolved(jitlink::AtomGraph &G) override; + + void notifyFinalized( + std::unique_ptr A) override; + + Error modifyPassConfig(const Triple &TT, + jitlink::PassConfiguration &Config) override; + + private: + TestResources &TR; + TestCaseFunction TestCase; + NotifyResolvedFunction NotifyResolved; + NotifyFinalizedFunction NotifyFinalized; + std::unique_ptr ObjBuffer; + std::unique_ptr MemMgr; + StringMap Externals; + }; + + JITLinkTestCommon(); + + std::unique_ptr + getTestResources(StringRef AsmSrc, StringRef Triple, bool PIC, + bool LargeCodeModel, MCTargetOptions Options) const { + return llvm::make_unique(AsmSrc, Triple, PIC, LargeCodeModel, + std::move(Options)); + } + + template + static Expected readInt(jitlink::AtomGraph &G, jitlink::DefinedAtom &A, + size_t Offset = 0) { + if (Offset + sizeof(T) > A.getContent().size()) + return make_error("Reading past end of atom content", + inconvertibleErrorCode()); + return support::endian::read(A.getContent().data() + Offset, + G.getEndianness()); + } + + template + static Expected readInt(jitlink::AtomGraph &G, StringRef AtomName, + size_t Offset = 0) { + auto DA = G.findDefinedAtomByName(AtomName); + if (!DA) + return DA.takeError(); + return readInt(G, *DA); + } + + static Expected> + disassemble(const MCDisassembler &Dis, jitlink::DefinedAtom &Atom, + size_t Offset = 0); + + static Expected decodeImmediateOperand(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, + size_t OpIdx, + size_t Offset = 0); + + static jitlink::Atom &atom(jitlink::AtomGraph &G, StringRef Name) { + return G.getAtomByName(Name); + } + + static jitlink::DefinedAtom &definedAtom(jitlink::AtomGraph &G, + StringRef Name) { + return G.getDefinedAtomByName(Name); + } + + static JITTargetAddress atomAddr(jitlink::AtomGraph &G, StringRef Name) { + return atom(G, Name).getAddress(); + } + + template + static size_t countEdgesMatching(jitlink::DefinedAtom &DA, + const PredT &Pred) { + return std::count_if(DA.edges().begin(), DA.edges().end(), Pred); + } + + template + static size_t countEdgesMatching(jitlink::AtomGraph &G, StringRef Name, + const PredT &Pred) { + return countEdgesMatching(definedAtom(G, Name), Pred); + } + +private: + + static bool AreTargetsInitialized; + void initializeLLVMTargets(); + + DenseMap Externals; +}; + +} // end namespace llvm + +#endif Index: llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp =================================================================== --- llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp +++ llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp @@ -0,0 +1,232 @@ +//===------- JITLinkTestCommon.cpp - Common code for JITLink tests --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "JITLinkTestCommon.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/Support/TargetSelect.h" + +using namespace llvm::jitlink; +namespace llvm { + +JITLinkTestCommon::TestResources::TestResources(StringRef AsmSrc, + StringRef TripleStr, bool PIC, + bool LargeCodeModel, + MCTargetOptions Options) + : ObjStream(ObjBuffer), Options(std::move(Options)) { + Triple TT(Triple::normalize(TripleStr)); + initializeTripleSpecifics(TT); + initializeTestSpecifics(AsmSrc, TT, PIC, LargeCodeModel); +} + +MemoryBufferRef +JITLinkTestCommon::TestResources::getTestObjectBufferRef() const { + return MemoryBufferRef(StringRef(ObjBuffer.data(), ObjBuffer.size()), + "Test object"); +} + +void JITLinkTestCommon::TestResources::initializeTripleSpecifics(Triple &TT) { + std::string Error; + TheTarget = TargetRegistry::lookupTarget("", TT, Error); + + if (!TheTarget) + report_fatal_error(Error); + + MRI.reset(TheTarget->createMCRegInfo(TT.getTriple())); + if (!MRI) + report_fatal_error("Could not build MCRegisterInfo for triple"); + + MAI.reset(TheTarget->createMCAsmInfo(*MRI, TT.getTriple())); + if (!MAI) + report_fatal_error("Could not build MCAsmInfo for triple"); + + MCII.reset(TheTarget->createMCInstrInfo()); + if (!MCII) + report_fatal_error("Could not build MCInstrInfo for triple"); + + STI.reset(TheTarget->createMCSubtargetInfo(TT.getTriple(), "", "")); + if (!STI) + report_fatal_error("Could not build MCSubtargetInfo for triple"); + + DisCtx = llvm::make_unique(MAI.get(), MRI.get(), nullptr); + Dis.reset(TheTarget->createMCDisassembler(*STI, *DisCtx)); + + if (!Dis) + report_fatal_error("Could not build MCDisassembler"); +} + +void JITLinkTestCommon::TestResources::initializeTestSpecifics( + StringRef AsmSrc, const Triple &TT, bool PIC, bool LargeCodeModel) { + SrcMgr.AddNewSourceBuffer(MemoryBuffer::getMemBuffer(AsmSrc), SMLoc()); + AsCtx = llvm::make_unique(MAI.get(), MRI.get(), &MOFI, &SrcMgr); + MOFI.InitMCObjectFileInfo(TT, PIC, *AsCtx, LargeCodeModel); + + std::unique_ptr CE( + TheTarget->createMCCodeEmitter(*MCII, *MRI, *AsCtx)); + if (!CE) + report_fatal_error("Could not build MCCodeEmitter"); + + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, Options)); + if (!MAB) + report_fatal_error("Could not build MCAsmBackend for test"); + + std::unique_ptr MOW(MAB->createObjectWriter(ObjStream)); + + MOS.reset(TheTarget->createMCObjectStreamer( + TT, *AsCtx, std::move(MAB), std::move(MOW), std::move(CE), *STI, + Options.MCRelaxAll, Options.MCIncrementalLinkerCompatible, false)); + + std::unique_ptr MAP( + createMCAsmParser(SrcMgr, *AsCtx, *MOS, *MAI)); + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *MAP, *MCII, Options)); + + if (!TAP) + report_fatal_error("Could not build MCTargetAsmParser for test"); + + MAP->setTargetParser(*TAP); + + if (MAP->Run(false)) + report_fatal_error("Failed to parse test case"); +} + +JITLinkTestCommon::TestJITLinkContext::TestJITLinkContext( + TestResources &TR, TestCaseFunction TestCase) + : TR(TR), TestCase(std::move(TestCase)) {} + +JITLinkTestCommon::TestJITLinkContext & +JITLinkTestCommon::TestJITLinkContext::setMemoryManager( + std::unique_ptr MM) { + assert(!MemMgr && "Memory manager already set"); + MemMgr = std::move(MM); + return *this; +} + +JITLinkMemoryManager & +JITLinkTestCommon::TestJITLinkContext::getMemoryManager() { + if (!MemMgr) + MemMgr = llvm::make_unique(); + return *MemMgr; +} + +MemoryBufferRef JITLinkTestCommon::TestJITLinkContext::getObjectBuffer() const { + return TR.getTestObjectBufferRef(); +} + +void JITLinkTestCommon::TestJITLinkContext::notifyFailed(Error Err) { + ADD_FAILURE() << "Unexpected failure: " << toString(std::move(Err)); +} + +void JITLinkTestCommon::TestJITLinkContext::lookup( + const DenseSet &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) { + jitlink::AsyncLookupResult LookupResult; + DenseSet MissingSymbols; + for (const auto &Symbol : Symbols) { + auto I = Externals.find(Symbol); + if (I != Externals.end()) + LookupResult[Symbol] = I->second; + else + MissingSymbols.insert(Symbol); + } + + if (MissingSymbols.empty()) + LookupContinuation(std::move(LookupResult)); + else { + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "Failed to resolve external symbols: ["; + for (auto &Sym : MissingSymbols) + ErrMsgStream << " " << Sym; + ErrMsgStream << " ]\n"; + } + LookupContinuation( + make_error(std::move(ErrMsg), inconvertibleErrorCode())); + } +} + +void JITLinkTestCommon::TestJITLinkContext::notifyResolved(AtomGraph &G) { + if (NotifyResolved) + NotifyResolved(G); +} + +void JITLinkTestCommon::TestJITLinkContext::notifyFinalized( + std::unique_ptr A) { + if (NotifyFinalized) + NotifyFinalized(std::move(A)); +} + +Error JITLinkTestCommon::TestJITLinkContext::modifyPassConfig( + const Triple &TT, PassConfiguration &Config) { + if (TestCase) + Config.PostFixupPasses.push_back([&](AtomGraph &G) -> Error { + TestCase(G); + return Error::success(); + }); + return Error::success(); +} + +JITLinkTestCommon::JITLinkTestCommon() { initializeLLVMTargets(); } + +Expected> +JITLinkTestCommon::disassemble(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, size_t Offset) { + ArrayRef InstBuffer( + reinterpret_cast(Atom.getContent().data()) + Offset, + Atom.getContent().size() - Offset); + + MCInst Inst; + uint64_t InstSize; + auto Status = + Dis.getInstruction(Inst, InstSize, InstBuffer, 0, nulls(), nulls()); + + if (Status != MCDisassembler::Success) + return make_error("Could not disassemble instruction", + inconvertibleErrorCode()); + + return std::make_pair(Inst, InstSize); +} + +Expected +JITLinkTestCommon::decodeImmediateOperand(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, + size_t OpIdx, size_t Offset) { + auto InstAndSize = disassemble(Dis, Atom, Offset); + if (!InstAndSize) + return InstAndSize.takeError(); + + if (OpIdx >= InstAndSize->first.getNumOperands()) + return make_error("Invalid operand index", + inconvertibleErrorCode()); + + auto &Op = InstAndSize->first.getOperand(OpIdx); + + if (!Op.isImm()) + return make_error("Operand at index is not immediate", + inconvertibleErrorCode()); + + return Op.getImm(); +} + +bool JITLinkTestCommon::AreTargetsInitialized = false; + +void JITLinkTestCommon::initializeLLVMTargets() { + if (!AreTargetsInitialized) { + InitializeAllTargets(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + InitializeAllAsmPrinters(); + InitializeAllDisassemblers(); + AreTargetsInitialized = true; + } +} + +} // end namespace llvm Index: llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTest_MachO_x86_64_Tests.cpp =================================================================== --- llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTest_MachO_x86_64_Tests.cpp +++ llvm/trunk/unittests/ExecutionEngine/JITLink/JITLinkTest_MachO_x86_64_Tests.cpp @@ -0,0 +1,224 @@ +//===---- JITLinkTest_MachO_x86_64.cpp - Tests for JITLink MachO/x86-64 ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "JITLinkTestCommon.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" +#include "llvm/Testing/Support/Error.h" + +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class JITLinkTest_MachO_x86_64 : public JITLinkTestCommon, + public testing::Test { +public: + using BasicVerifyGraphFunction = + std::function; + + void runBasicVerifyGraphTest(StringRef AsmSrc, StringRef Triple, + StringMap Externals, + bool PIC, bool LargeCodeModel, + MCTargetOptions Options, + BasicVerifyGraphFunction RunGraphTest) { + auto TR = getTestResources(AsmSrc, Triple, PIC, LargeCodeModel, + std::move(Options)); + + auto JTCtx = llvm::make_unique( + *TR, [&](AtomGraph &G) { RunGraphTest(G, TR->getDisassembler()); }); + + JTCtx->externals() = std::move(Externals); + + jitLink_MachO_x86_64(std::move(JTCtx)); + } + +protected: + static void verifyIsPointerTo(AtomGraph &G, DefinedAtom &A, Atom &Target) { + EXPECT_EQ(A.edges_size(), 1U) << "Incorrect number of edges for pointer"; + if (A.edges_size() != 1U) + return; + auto &E = *A.edges().begin(); + EXPECT_EQ(E.getKind(), Pointer64) + << "Expected pointer to have a pointer64 relocation"; + EXPECT_EQ(&E.getTarget(), &Target) << "Expected edge to point at target"; + EXPECT_THAT_EXPECTED(readInt(G, A), HasValue(Target.getAddress())) + << "Pointer does not point to target"; + } + + static void verifyGOTLoad(AtomGraph &G, DefinedAtom &A, Edge &E, + Atom &Target) { + EXPECT_EQ(E.getAddend(), 0U) << "Expected GOT load to have a zero addend"; + EXPECT_TRUE(E.getTarget().isDefined()) + << "GOT entry should be a defined atom"; + if (!E.getTarget().isDefined()) + return; + + verifyIsPointerTo(G, static_cast(E.getTarget()), Target); + } + + static void verifyCall(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + EXPECT_EQ(E.getKind(), Branch32) << "Edge is not a Branch32"; + EXPECT_EQ(E.getAddend(), 0U) << "Expected no addend on stub call"; + EXPECT_EQ(&E.getTarget(), &Callee) + << "Edge does not point at expected callee"; + + JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + uint64_t PCRelDelta = Callee.getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED( + decodeImmediateOperand(Dis, Caller, 0, E.getOffset() - 1), + HasValue(PCRelDelta)); + } + + static void verifyIndirectCall(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + EXPECT_EQ(E.getKind(), PCRel32) << "Edge is not a PCRel32"; + EXPECT_EQ(E.getAddend(), 0) << "Expected no addend on stub cal"; + EXPECT_TRUE(E.getTarget().isDefined()) << "Target is not a defined atom"; + if (!E.getTarget().isDefined()) + return; + verifyIsPointerTo(G, static_cast(E.getTarget()), Callee); + + JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + uint64_t PCRelDelta = E.getTarget().getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED( + decodeImmediateOperand(Dis, Caller, 3, E.getOffset() - 2), + HasValue(PCRelDelta)); + } + + static void verifyCallViaStub(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + verifyCall(Dis, G, Caller, E, E.getTarget()); + + if (!E.getTarget().isDefined()) { + ADD_FAILURE() << "Edge target is not a stub"; + return; + } + + auto &StubAtom = static_cast(E.getTarget()); + EXPECT_EQ(StubAtom.edges_size(), 1U) + << "Expected one edge from stub to target"; + + auto &StubEdge = *StubAtom.edges().begin(); + + verifyIndirectCall(Dis, G, static_cast(StubAtom), StubEdge, + Callee); + } +}; + +} // end anonymous namespace + +// Test each operation on LegacyObjectTransformLayer. +TEST_F(JITLinkTest_MachO_x86_64, BasicRelocations) { + runBasicVerifyGraphTest( + R"( + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 10, 14 + .globl _bar + .p2align 4, 0x90 + _bar: + callq _baz + + .globl _foo + .p2align 4, 0x90 + _foo: + callq _bar + _foo.1: + movq _y@GOTPCREL(%rip), %rcx + _foo.2: + movq _p(%rip), %rdx + + .section __DATA,__data + .globl _x + .p2align 2 + _x: + .long 42 + + .globl _p + .p2align 3 + _p: + .quad _x + + .subsections_via_symbols)", + "x86_64-apple-macosx10.14", + {{"_y", JITEvaluatedSymbol(0xdeadbeef, JITSymbolFlags::Exported)}, + {"_baz", JITEvaluatedSymbol(0xcafef00d, JITSymbolFlags::Exported)}}, + true, false, MCTargetOptions(), + [](AtomGraph &G, const MCDisassembler &Dis) { + // Name the atoms in the asm above. + auto &Baz = atom(G, "_baz"); + auto &Y = atom(G, "_y"); + + auto &Bar = definedAtom(G, "_bar"); + auto &Foo = definedAtom(G, "_foo"); + auto &Foo_1 = definedAtom(G, "_foo.1"); + auto &Foo_2 = definedAtom(G, "_foo.2"); + auto &X = definedAtom(G, "_x"); + auto &P = definedAtom(G, "_p"); + + // Check unsigned reloc for _p + { + EXPECT_EQ(P.edges_size(), 1U) << "Unexpected number of relocations"; + EXPECT_EQ(P.edges().begin()->getKind(), Pointer64) + << "Unexpected edge kind for _p"; + EXPECT_THAT_EXPECTED(readInt(G, P), + HasValue(X.getAddress())) + << "Unsigned relocation did not apply correctly"; + } + + // Check that _bar is a call-via-stub to _baz. + // This will check that the call goes to a stub, that the stub is an + // indirect call, and that the pointer for the indirect call points to + // baz. + { + EXPECT_EQ(Bar.edges_size(), 1U) + << "Incorrect number of edges for bar"; + EXPECT_EQ(Bar.edges().begin()->getKind(), Branch32) + << "Unexpected edge kind for _bar"; + verifyCallViaStub(Dis, G, Bar, *Bar.edges().begin(), Baz); + } + + // Check that _foo is a direct call to _bar. + { + EXPECT_EQ(Foo.edges_size(), 1U) + << "Incorrect number of edges for foo"; + EXPECT_EQ(Foo.edges().begin()->getKind(), Branch32); + verifyCall(Dis, G, Foo, *Foo.edges().begin(), Bar); + } + + // Check .got load in _foo.1 + { + EXPECT_EQ(Foo_1.edges_size(), 1U) + << "Incorrect number of edges for foo_1"; + EXPECT_EQ(Foo_1.edges().begin()->getKind(), PCRel32); + verifyGOTLoad(G, Foo_1, *Foo_1.edges().begin(), Y); + } + + // Check PCRel ref to _p in _foo.2 + { + EXPECT_EQ(Foo_2.edges_size(), 1U) + << "Incorrect number of edges for foo_2"; + EXPECT_EQ(Foo_2.edges().begin()->getKind(), PCRel32); + + JITTargetAddress FixupAddress = + Foo_2.getAddress() + Foo_2.edges().begin()->getOffset(); + uint64_t PCRelDelta = P.getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED(decodeImmediateOperand(Dis, Foo_2, 4, 0), + HasValue(PCRelDelta)) + << "PCRel load does not reference expected target"; + } + }); +}