Index: include/llvm/ExecutionEngine/JITLink/JITLink.h =================================================================== --- /dev/null +++ include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -0,0 +1,504 @@ +//===------------ JITLink.h - JIT linker functionality ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains generic JIT-linker types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Memory.h" + +#include +#include +#include + +namespace llvm { + +class MemoryBuffer; + +namespace jitlink { + +/// Base class for errors originating in JIT linker, e.g. missing relocation +/// support. +class JITLinkError : public ErrorInfo { +public: + static char ID; + + JITLinkError(Twine ErrMsg) : ErrMsg(ErrMsg.str()) {} + + void log(raw_ostream &OS) const override; + const std::string &getErrorMessage() const { return ErrMsg; } + std::error_code convertToErrorCode() const override; + +private: + std::string ErrMsg; +}; + +/// Manages allocations of JIT memory. +/// +/// Instances of this class may be accessed concurrently from multiple threads +/// and their implemetations should include any necessary synchronization. +class JITLinkMemoryManager { +public: + using ProtectionFlags = sys::Memory::ProtectionFlags; + + class SegmentRequest { + public: + SegmentRequest() = default; + SegmentRequest(size_t Size, unsigned Alignment) + : Size(Size), Alignment(Alignment) {} + size_t getSize() const { return Size; } + unsigned getAlignment() const { return Alignment; } + + private: + size_t Size = 0; + unsigned Alignment = 0; + }; + + using SegmentsRequestMap = DenseMap; + + using FinalizeContinuation = std::function; + + /// Represents an allocation created by the memory manager. + /// + /// An allocation object is responsible for allocating and owning jit-linker + /// working and target memory, and for transfering from working to target + /// memory. + /// + class Allocation { + public: + virtual ~Allocation(); + + /// Should return the address of linker working memory for the segment with + /// the given protection flags. + virtual MutableArrayRef getWorkingMemory(ProtectionFlags Seg) = 0; + + /// Should return the final address in the target process where the segment + /// will reside. + virtual JITTargetAddress getTargetMemory(ProtectionFlags Seg) = 0; + + /// Should transfer from working memory to target memory, and release + /// working memory. + virtual void finalizeAsync(FinalizeContinuation OnFinalize) = 0; + + /// Should deallocate target memory. + virtual Error deallocate() = 0; + }; + + virtual ~JITLinkMemoryManager(); + + /// Create an Allocation object. + virtual Expected> + allocate(const SegmentsRequestMap &Request) = 0; +}; + +// Forward declare the Atom class. +class Atom; + +/// Edge class. Represents both object file relocations, as well as layout and +/// keep-alive constraints. +class Edge { +public: + using Kind = uint8_t; + + using GenericEdgeKind = enum : Kind { + Invalid, + LayoutNext, // Lays target out after current atom. Offset/Addend zero. + FirstKeepAlive, // First edge kind that preserves liveness. + KeepAlive = FirstKeepAlive, // Keeps target alive. Offset/addend zero. + FirstRelocation // First architecture specific relocation. + }; + + using OffsetT = uint32_t; + using AddendT = int64_t; + + Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} + + OffsetT getOffset() const { return Offset; } + Kind getKind() const { return K; } + void setKind(Kind K) { this->K = K; } + bool isRelocation() const { return K >= FirstRelocation; } + Kind getRelocation() const { + assert(isRelocation() && "Not a relocation edge"); + return K - FirstRelocation; + } + bool isKeepAlive() const { return K >= FirstKeepAlive; } + Atom &getTarget() const { return *Target; } + void setTarget(Atom &Target) { this->Target = &Target; } + AddendT getAddend() const { return Addend; } + +private: + Atom *Target; + OffsetT Offset; + AddendT Addend; + Kind K = 0; +}; + +using EdgeVector = std::vector; + +const StringRef getGenericEdgeKindName(Edge::Kind K); + +/// Base Atom class. Used by absolute and undefined atoms. +class Atom { + friend class AtomGraph; + +protected: + Atom(StringRef Name) + : Name(Name), IsLive(false), IsDefined(false), IsResolved(false) {} + Atom(StringRef Name, JITTargetAddress Address) + : Name(Name), Address(Address), IsLive(false), IsDefined(true), + IsResolved(true) {} + +public: + StringRef getName() const { return Name; } + JITTargetAddress getAddress() const { return Address; } + void setAddress(JITTargetAddress Address) { + this->Address = Address; + IsResolved = true; + } + bool isDefined() const { return IsDefined; } + bool isResolved() const { return IsResolved; } + bool isLive() const { return IsLive; } + void setIsLive(bool IsLive) { this->IsLive = IsLive; } + +private: + StringRef Name; + JITTargetAddress Address = 0; + bool IsLive : 1; + bool IsDefined : 1; + bool IsResolved : 1; +}; + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A); +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName); + +/// Represents an object file section. +class Section { + friend class AtomGraph; + +private: + Section(StringRef Name, sys::Memory::ProtectionFlags Prot, uint32_t Alignment, + unsigned Ordinal) + : Name(Name), Prot(Prot), Alignment(Alignment), Ordinal(Ordinal) {} + +public: + StringRef getName() const { return Name; } + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } + uint32_t getAlignment() const { return Alignment; } + unsigned getSectionOrdinal() const { return Ordinal; } + size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } + +private: + StringRef Name; + sys::Memory::ProtectionFlags Prot; + uint32_t Alignment = 0; + unsigned Ordinal = 0; + unsigned NextAtomOrdinal = 0; +}; + +/// Defined atom class. Suitable for use by defined named and anonymous +/// atoms. +class DefinedAtom : public Atom { + friend class AtomGraph; + +private: + DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) + : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), + Alignment(Alignment) {} + + DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, + uint32_t Alignment) + : Atom(Name, Address), Parent(Parent), + Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) {} + +public: + using edge_iterator = EdgeVector::iterator; + + Section &getSection() const { return Parent; } + + StringRef getContent() const { return Content; } + void setContent(StringRef Content) { this->Content = Content; } + + uint32_t getAlignment() const { return Alignment; } + + void addEdge(Edge::Kind K, JITTargetAddress Offset, Atom &Target, + JITTargetAddress Addend) { + Edges.push_back(Edge(K, Offset, Target, Addend)); + } + + iterator_range edges() { + return make_range(Edges.begin(), Edges.end()); + } + size_t edges_size() const { return Edges.size(); } + bool edges_empty() const { return Edges.empty(); } + + unsigned getOrdinal() const { return Ordinal; } + +private: + Section &Parent; + StringRef Content; + unsigned Ordinal = 0; + uint32_t Alignment = 0; + EdgeVector Edges; +}; + +class AtomGraph { +public: + using AddressToAtomMap = std::map; + using NamedAtomMap = DenseMap; + using ExternalAtomSet = DenseSet; + using DefinedAtomSet = DenseSet; + using external_atom_iterator = ExternalAtomSet::iterator; + using defined_atom_iterator = DefinedAtomSet::iterator; + + AtomGraph(unsigned PointerSize, support::endianness Endianness) + : PointerSize(PointerSize), Endianness(Endianness) {} + + /// Returns the pointer size for use in this graph. + unsigned getPointerSize() const { return PointerSize; } + + /// Returns the endianness of atom-content in this graph. + support::endianness getEndianness() const { return Endianness; } + + /// Create a section with the given name, protection flags, and alignment. + Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot, + uint32_t Alignment) { + std::unique_ptr
Sec( + new Section(Name, Prot, Alignment, Sections.size())); + Sections.push_back(std::move(Sec)); + return *Sections.back(); + } + + /// Add an external atom representing an undefined symbol in this graph. + Atom &addExternalAtom(StringRef Name) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + Atom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); + new (A) Atom(Name); + ExternalAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + /// Add an external atom representing an absolute symbol. + Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { + Atom &A = addExternalAtom(Name); + A.setAddress(Addr); + return A; + } + + DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, + uint32_t Alignment) { + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Address, Alignment); + getAddrToAtomMap()[A->getAddress()] = A; + DefinedAtoms.insert(A); + return *A; + } + + DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, + JITTargetAddress Address, uint32_t Alignment) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Name, Address, Alignment); + getAddrToAtomMap()[A->getAddress()] = A; + DefinedAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + iterator_range external_atoms() { + return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + } + + iterator_range defined_atoms() { + return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + } + + bool defined_atoms_empty() const { return DefinedAtoms.empty(); } + + /// Returns the atom with the given name, which must exist in this graph. + Atom &getAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); + return *I->second; + } + + /// Returns the atom with the given name, which must exist in this graph and + /// be a DefinedAtom. + DefinedAtom &getDefinedAtomByName(StringRef Name) { + auto &A = getAtomByName(Name); + assert(A.isDefined() && "Atom is not a defined atom"); + return static_cast(A); + } + + /// Search for the given atom by name. + /// Returns the atom (if found) or an error (if no atom with this name + /// exists). + Expected findAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + return *I->second; + } + + /// Search for the given defined atom by name. + /// Returns the defined atom (if found) or an error (if no atom with this + /// name exists, or if one exists but is not a defined atom). + Expected findDefinedAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + if (!I->second->isDefined()) + return make_error("Atom " + Name + + " exists but is not a " + "defined atom"); + return static_cast(*I->second); + } + + /// Returns the atom covering the given address, or an error if no such atom + /// exists. + /// + /// Returns null if no atom exists at the given address. + DefinedAtom *getAtomByAddress(JITTargetAddress Address) { + refreshAddrToAtomCache(); + + // If there are no defined atoms, bail out early. + if (AddrToAtomCache->empty()) + return nullptr; + + // Find the atom *after* the given address. + auto I = AddrToAtomCache->upper_bound(Address); + + // If this address falls before any known atom, bail out. + if (I == AddrToAtomCache->begin()) + return nullptr; + + // The atom we're looking for is the one before the atom we found. + --I; + + // Otherwise range check the atom that was found. + assert(!I->second->getContent().empty() && "Atom content not set"); + if (Address >= I->second->getAddress() + I->second->getContent().size()) + return nullptr; + + return I->second; + } + + /// Like getAtomByAddress, but returns an Error if the given address is not + /// covered by an atom, rather than a null pointer. + Expected findAtomByAddress(JITTargetAddress Address) { + if (auto *DA = getAtomByAddress(Address)) + return *DA; + return make_error("No atom at address " + + formatv("{0:x16}", Address)); + } + + /// Remove the given defined atom from the graph. + void removeAtom(DefinedAtom &A) { + assert(DefinedAtoms.count(&A) && "Atom is not in the DefinedAtoms map"); + if (AddrToAtomCache) + AddrToAtomCache->erase(A.getAddress()); + DefinedAtoms.erase(&A); + A.~DefinedAtom(); + } + + /// Invalidate the atom-to-address map. + void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + + /// Dump the graph. + /// + /// If supplied, the EdgeKindToName function will be used to name edge + /// kinds in the debug output. Otherwise raw edge kind numbers will be + /// displayed. + void dump(raw_ostream &OS, + std::function EdegKindToName = + std::function()); + +private: + AddressToAtomMap &getAddrToAtomMap() { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + const AddressToAtomMap &getAddrToAtomMap() const { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + void refreshAddrToAtomCache() const { + if (!AddrToAtomCache) { + AddrToAtomCache = AddressToAtomMap(); + for (auto *DA : DefinedAtoms) + (*AddrToAtomCache)[DA->getAddress()] = DA; + } + } + + unsigned PointerSize; + support::endianness Endianness; + std::vector> Sections; + BumpPtrAllocator AtomAllocator; + NamedAtomMap NamedAtoms; + ExternalAtomSet ExternalAtoms; + DefinedAtomSet DefinedAtoms; + mutable Optional AddrToAtomCache; +}; + +/// A function for mutating AtomGraphs. +using AtomGraphPassFunction = std::function; + +/// A JITLinkMemoryManager that allocates in-process memory. +class InProcessMemoryManager : public JITLinkMemoryManager { +public: + Expected> + allocate(const SegmentsRequestMap &Request) override; +}; + +/// Callback to enable inspection of a fixed-up atom graph, including +/// inspecting assigned atom addresses and contents. +using OnResolveFunction = std::function)>; + +using OnFinalizeFunction = std::function>)>; + +/// A map of symbol names to resolved addresses. +using AsyncLookupResult = DenseMap; + +/// A function to call with a resolved symbol map (See AsyncLookupResult) or an +/// error if resolution failed. +using JITLinkAsyncLookupContinuation = + std::function LR)>; + +/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) +/// for the given symbols, calling the given continuation with either the result +/// (if the lookup succeeds), or an error (if the lookup fails). +using JITLinkAsyncLookupFunction = + std::function &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation)>; + +/// Marks all atoms in a graph live. This can be used as a default, conservative +/// mark-live implementation. +Error markAllAtomsLive(AtomGraph &G); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H Index: include/llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h =================================================================== --- /dev/null +++ include/llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h @@ -0,0 +1,56 @@ +//===--- JITLink_MachO_x86_64.h - JIT link functions for MachO --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_x86_64_Edges { + +enum MachOX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Pointer64, + Pointer64Anon, + PCRel32, + PCRel32Minus1, + PCRel32Minus2, + PCRel32Minus4, + PCRel32Anon, + PCRel32Minus1Anon, + PCRel32Minus2Anon, + PCRel32Minus4Anon, + PCRel32GOTLoad, + PCRel32GOT, + PCRel32TLV, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_x86_64_Edges + +void jitLink_MachO_x86_64( + std::unique_ptr ObjBuffer, JITLinkMemoryManager &MemMgr, + JITLinkAsyncLookupFunction Lookup, OnResolveFunction OnResolve, + OnFinalizeFunction OnFinalize, + AtomGraphPassFunction MarkLivePass = AtomGraphPassFunction(), + AtomGraphPassFunction GOTAndStubsPass = AtomGraphPassFunction()); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_MACHO_X86_64_H Index: include/llvm/Support/Memory.h =================================================================== --- include/llvm/Support/Memory.h +++ include/llvm/Support/Memory.h @@ -18,6 +18,10 @@ #include namespace llvm { + +// Forward declare raw_ostream: it is used for debug dumping below. +class raw_ostream; + namespace sys { /// This class encapsulates the notion of a memory block which has an address @@ -138,7 +142,14 @@ MemoryBlock M; }; -} -} +#ifndef NDEBUG + /// Debugging output for Memory::ProtectionFlags. + raw_ostream &operator<<(raw_ostream &OS, const Memory::ProtectionFlags &PF); + + /// Debugging output for MemoryBlock. + raw_ostream &operator<<(raw_ostream &OS, const MemoryBlock &MB); +#endif // ifndef NDEBUG + } // end namespace sys + } // end namespace llvm #endif Index: lib/ExecutionEngine/CMakeLists.txt =================================================================== --- lib/ExecutionEngine/CMakeLists.txt +++ lib/ExecutionEngine/CMakeLists.txt @@ -19,6 +19,7 @@ endif() add_subdirectory(Interpreter) +add_subdirectory(JITLink) add_subdirectory(MCJIT) add_subdirectory(Orc) add_subdirectory(RuntimeDyld) Index: lib/ExecutionEngine/JITLink/CMakeLists.txt =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -0,0 +1,9 @@ +add_llvm_library(LLVMJITLink + JITLink.cpp + JITLinkGeneric.cpp + JITLink_MachO_x86_64.cpp + MachOAtomGraphBuilder.cpp + + DEPENDS + intrinsics_gen + ) Index: lib/ExecutionEngine/JITLink/JITLink.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,198 @@ +//===--------- JITLinkerMachO.cpp - Run-time JIT linker for MachO ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; + +JITLinkMemoryManager::Allocation::~Allocation() = default; + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { + if (A.getName().empty()) + return OS << "<" << &A << ": anon@" + << format("0x%016" PRIx64, A.getAddress()) << ">"; + return OS << "<" << &A << ": " << A.getName() << ">"; +} + +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) + << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName + << " -> " << E.getTarget() << " + " << E.getAddend(); +} + +void AtomGraph::dump(raw_ostream &OS, + std::function EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Defined atoms:\n"; + for (auto *DA : defined_atoms()) { + OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + << "\n"; + for (auto &E : DA->edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, *DA, E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, *DA, E, EdgeNumberString); + } + OS << "\n"; + } + } + OS << "External atoms:\n"; + for (auto *A : external_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; +} + +const StringRef getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + case Edge::LayoutNext: + return "Layout-Next"; + default: + llvm_unreachable("Unrecognized relocation kind"); + } +} + +Expected> +InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast(SegBlocks[Seg].base()), + SegBlocks[Seg].size()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return reinterpret_cast(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + for (auto &KV : SegBlocks) + if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot && sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + for (auto &KV : Request) { + auto &Seg = KV.second; + + if (Seg.getAlignment() > sys::Process::getPageSize()) + return make_error("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + if (sys::Process::getPageSize() % Seg.getAlignment() != 0) + return make_error("Page size is not a multiple of " + "alignment", + inconvertibleErrorCode()); + + std::error_code EC; + Blocks[KV.first] = sys::Memory::allocateMappedMemory(Seg.getSize(), nullptr, + ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + } + return std::unique_ptr( + new IPMMAlloc(std::move(Blocks))); +} + +Error markAllAtomsLive(AtomGraph &G) { + for (auto *DA : G.defined_atoms()) + DA->setIsLive(true); + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm Index: lib/ExecutionEngine/JITLink/JITLinkGeneric.h =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,232 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr ObjBuffer, + JITLinkMemoryManager &MemMgr, JITLinkAsyncLookupFunction Lookup, + OnResolveFunction OnResolved, OnFinalizeFunction OnFinalized) + : ObjBuffer(std::move(ObjBuffer)), MemMgr(MemMgr), + Lookup(std::move(Lookup)), OnResolved(std::move(OnResolved)), + OnFinalized(std::move(OnFinalized)) { + assert(this->ObjBuffer && "ObjBuffer can not be null"); + assert(this->Lookup && "Lookup must be set"); + assert(this->OnResolved && "OnResolved must be set"); + assert(this->OnFinalized && "OnFinalized must be set"); + } + + virtual ~JITLinkerBase(); + +protected: + using SegmentAtomOrder = DenseMap>; + + // Phase 1: + // 1.1: Build atom graph + // 1.2: Identify root atoms and dead-strip graph + // 1.3: Run post-dead-strip passes + // 1.4: Sort atoms into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up atom contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr Self, + Expected LookupResult); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr Self, Error Err); + + // Build a graph from the given object buffer. + // To be implemented by the client. + virtual Expected> + buildGraph(MemoryBufferRef ObjBuffer) = 0; + + // Run pre-dead-strip passes on the graph. + // + // This method is called on the graph after it is built, and before any atoms + // have been dead stripped. The most important task for these passes is to + // mark the dead strip roots for the graph. + virtual Error runPreDeadStripPasses(AtomGraph &G) = 0; + + // Run post-dead-strip passes on the graph. + // + // This method is called on the graph after dead atoms have been stripped, + // and provides subclasses with an opportunity to inspect and modify the + // graph. In particular, it is intended to enable GOT and Stub atom insertion + // for architectures that require it. + virtual Error runPostDeadStripPasses(AtomGraph &G) = 0; + + // Copy atom contents and apply relocations. + // Implemented in JITLinker. + virtual Error + copyAndFixUpAllAtoms(const SegmentAtomOrder &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; + + // For debug dumping of the atom graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + +private: + void sortAtoms(); + Error allocateSegments(const SegmentAtomOrder &SegAtoms); + DenseSet identifyUnresolvedExternals() const; + void applyLookupResult(AsyncLookupResult LR); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr ObjBuffer; + JITLinkMemoryManager &MemMgr; + JITLinkAsyncLookupFunction Lookup; + OnResolveFunction OnResolved; + OnFinalizeFunction OnFinalized; + std::unique_ptr G; + SegmentAtomOrder Layout; + std::unique_ptr Alloc; +}; + +template class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template static void link(ArgTs &&... Args) { + auto L = llvm::make_unique(std::forward(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast(*this); + } + + Error + copyAndFixUpAllAtoms(const SegmentAtomOrder &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &Atoms = KV.second; + + auto SegMem = Alloc.getWorkingMemory( + static_cast(Prot)); + char *LastAtomEnd = SegMem.data(); + char *AtomDataPtr = nullptr; + + LLVM_DEBUG(dbgs() << " Processing segment " + << static_cast(Prot) + << " [ " << (void *)SegMem.data() << " .. " + << (void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n";); + + for (auto *A : Atoms) { + AtomDataPtr = LastAtomEnd; + + // Align. + AtomDataPtr += alignmentAdjustment(AtomDataPtr, A->getAlignment()); + LLVM_DEBUG(dbgs() << " Bumped atom pointer to " + << (void *)AtomDataPtr << " to meet alignment of " + << A->getAlignment() << "\n";); + + // Zero pad up to alignment. + LLVM_DEBUG(if (LastAtomEnd != AtomDataPtr) dbgs() + << " Zero padding from " << (void *)LastAtomEnd + << " to " << (void *)AtomDataPtr << "\n";); + while (LastAtomEnd != AtomDataPtr) + *LastAtomEnd++ = 0; + + // Copy initial atom content. + LLVM_DEBUG(dbgs() << " Copying atom " << *A << " content, " + << A->getContent().size() << " bytes, from " + << (void *)A->getContent().data() << " to " + << (void *)AtomDataPtr << "\n";); + memcpy(AtomDataPtr, A->getContent().data(), A->getContent().size()); + + // Copy atom data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : A->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*A, E, AtomDataPtr)) + return Err; + } + + // Point the atom's content to the fixed up buffer. + A->setContent(StringRef(AtomDataPtr, A->getContent().size())); + + // Update atom end pointer. + LastAtomEnd = AtomDataPtr + A->getContent().size(); + } + + // Zero pad the rest of the segment. + LLVM_DEBUG(dbgs() << " Zero padding end of segment from " + << (void *)LastAtomEnd << " to " + << (void *)((char *)SegMem.data() + SegMem.size()) + << "\n";); + while (LastAtomEnd != SegMem.data() + SegMem.size()) + *LastAtomEnd++ = 0; + } + + return Error::success(); + } +}; + +void deadStrip(AtomGraph &G); + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H Index: lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,453 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; + +namespace { + +class EHFrameParser { +public: + EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), + EHFrameAddress(EHFrameAddress), + EHFrameReader(EHFrameContent, G.getEndianness()), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + + Error atomize() { + while (!EHFrameReader.empty()) { + size_t RecordOffset = EHFrameReader.getOffset(); + + LLVM_DEBUG( + dbgs() << "Processing eh-frame record at " + << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) + << " (offset " << RecordOffset << ")\n";); + + size_t CIELength = 0; + uint32_t CIELengthField; + if (auto Err = EHFrameReader.readInteger(CIELengthField)) + return Err; + + // Process CIE length/extended-length fields to build the atom. + // + // The value of these fields describe the length of the *rest* of the CIE + // (not including data up to the end of the field itself) so we have to + // bump CIELength to include the data up to the end of the field: 4 bytes + // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. + if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + break; + + // If the regular length field's value is 0xffffffff, use extended length. + if (CIELengthField == 0xffffffff) { + uint64_t CIEExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + return Err; + if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + if (CIEExtendedLengthField + 12 > std::numeric_limits::max()) + return make_error("CIE record too large to process"); + CIELength = CIEExtendedLengthField + 12; + } else { + if (CIELengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + CIELength = CIELengthField + 4; + } + + LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); + + // Add an atom for this record. + CurRecordAtom = &G.addAnonymousAtom( + EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); + CurRecordAtom->setContent( + EHFrameContent.substr(EHFrameReader.getOffset(), CIELength)); + + // Read the CIE Pointer. + size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); + uint32_t CIEPointer; + if (auto Err = EHFrameReader.readInteger(CIEPointer)) + return Err; + + // Based on the CIE pointer value, parse this as a CIE or FDE record. + if (CIEPointer == 0) { + if (auto Err = processCIE()) + return Err; + } else { + if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + return Err; + } + + EHFrameReader.setOffset(RecordOffset + CIELength); + } + + return Error::success(); + } + +private: + Error processCIE() { + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + MostRecentCIE = CurRecordAtom; + + // TODO: Verify augmentation string. + + return Error::success(); + } + + Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + // Sanity check the CIE pointer: if this is an FDE it must be proceeded by + // a CIE. + if (MostRecentCIE == nullptr) + return make_error("__eh_frame must start with CIE, not " + "FDE"); + + LLVM_DEBUG(dbgs() << " CIE pointer: " + << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) + << "\n";); + + // Verify that this FDE's CIE pointer points to the most recent CIE entry. + if (CIEPointerAddress - CIEPointer != MostRecentCIE->getAddress()) + return make_error("__eh_frame FDE's CIE Pointer does not " + "point at the most recent CIE"); + + // The CIEPointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToCIERelocKind, + CIEPointerAddress - CurRecordAtom->getAddress(), + *MostRecentCIE, 0); + + // Read and sanity check the PC-start pointer. + JITTargetAddress PCTargetDeltaAddress = + EHFrameAddress + EHFrameReader.getOffset(); + JITTargetAddress PCTargetDelta; + + if (G.getPointerSize() == 8) { + if (auto Err = EHFrameReader.readInteger(PCTargetDelta)) + return Err; + } else if (G.getPointerSize() == 4) { + uint32_t PCTargetDelta32; + if (auto Err = EHFrameReader.readInteger(PCTargetDelta32)) + return Err; + PCTargetDelta = PCTargetDelta32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + + JITTargetAddress PCTargetAddr = PCTargetDeltaAddress + PCTargetDelta; + LLVM_DEBUG(dbgs() << " PC start: " << format("0x%016" PRIx64, PCTargetAddr) + << "\n"); + + auto *TargetAtom = G.getAtomByAddress(PCTargetAddr); + + if (!TargetAtom) + return make_error("FDE PC-start does not point at atom"); + + if (TargetAtom->getAddress() != PCTargetAddr) + return make_error("FDE PC-start should point at start of " + "atom"); + + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + + // The PC-start pointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + PCTargetDeltaAddress - CurRecordAtom->getAddress(), + *TargetAtom, 0); + // Add a keep-alive relocation from the function to the FDE to ensure it is + // not dead stripped. + TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + + return Error::success(); + } + + AtomGraph &G; + Section &EHFrameSection; + StringRef EHFrameContent; + JITTargetAddress EHFrameAddress; + BinaryStreamReader EHFrameReader; + DefinedAtom *CurRecordAtom = nullptr; + DefinedAtom *MostRecentCIE = nullptr; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; +}; + +} // namespace + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr Self) { + + // Build the atom graph. + if (auto GraphOrErr = buildGraph(ObjBuffer->getMemBufferRef())) + G = std::move(*GraphOrErr); + else + return OnResolved(GraphOrErr.takeError()); + assert(G && "Graph should have been created by buildGraph above"); + + // Dead-strip and optimize the graph. + if (auto Err = runPreDeadStripPasses(*G)) + return OnResolved(std::move(Err)); + + deadStrip(*G); + + LLVM_DEBUG(dbgs() << "Atom graph post-pruning:\n"; dumpGraph(dbgs());); + + // Run post-pruning passes. + if (auto Err = runPostDeadStripPasses(*G)) + return OnResolved(std::move(Err)); + + // Sort atoms into segments. + sortAtoms(); + assert((Layout.empty() == G->defined_atoms_empty()) && + "Layout should not be empty unless graph is empty"); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return OnResolved(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase2Continuation = + [UnownedSelf]( + Expected> LookupResult) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); + }; + + Lookup(identifyUnresolvedExternals(), std::move(Phase2Continuation)); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr Self, + Expected LR) { + // If the lookup failed, bail out. + if (!LR) + return OnResolved(LR.takeError()); + + // Assign addresses to external atoms. + applyLookupResult(*LR); + + LLVM_DEBUG(dbgs() << "Atom graph before copy-and-fixup:\n"; + dumpGraph(dbgs());); + + // Copy atom content to working memory and fix up. + if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + return OnResolved(std::move(Err)); + + LLVM_DEBUG(dbgs() << "Atom graph after copy-and-fixup:\n"; + dumpGraph(dbgs());); + + OnResolved(*G); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr Self, Error Err) { + if (Err) + return OnFinalized(std::move(Err)); + OnFinalized(std::move(Alloc)); +} + +void JITLinkerBase::sortAtoms() { + // Group atoms by segment. + for (auto *DA : G->defined_atoms()) + Layout[DA->getSection().getProtectionFlags()].push_back(DA); + + // Order atoms within each segment: first by section ordinal, then by atom + // ordinal. + for (auto &KV : Layout) + std::sort(KV.second.begin(), KV.second.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + auto &LHSP = LHS->getSection(); + auto &RHSP = RHS->getSection(); + if (LHSP.getSectionOrdinal() < RHSP.getSectionOrdinal()) + return true; + if (LHSP.getSectionOrdinal() > RHSP.getSectionOrdinal()) + return false; + return LHS->getOrdinal() < RHS->getOrdinal(); + }); + + LLVM_DEBUG( + dbgs() << "Segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast(KV.first) << ":\n"; + for (auto *DA : KV.second) + dbgs() << " " << *DA << "\n"; + } + ); +} + +Error JITLinkerBase::allocateSegments(const SegmentAtomOrder &SegAtoms) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : SegAtoms) { + auto &Prot = KV.first; + auto &Atoms = KV.second; + + size_t SegSize = 0; + for (auto *A : Atoms) { + SegSize = alignTo(SegSize, A->getAlignment()); + SegSize += A->getContent().size(); + } + + uint32_t SegAlign = std::max(Atoms.front()->getAlignment(), + Atoms.front()->getSection().getAlignment()); + + if (SegAlign % Atoms.front()->getAlignment() != 0) + return make_error("Segment alignment does not accommodate " + "first atom alignment"); + + Segments[Prot] = {SegSize, SegAlign}; + + LLVM_DEBUG(dbgs() << (&KV == &*SegAtoms.begin() ? "" : "; ") + << static_cast(Prot) << ": " + << SegSize << " bytes, align " << SegAlign << "";); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = MemMgr.allocate(Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG(dbgs() << "JIT linker got working memory:\n"; for (auto &KV + : SegAtoms) { + auto Prot = static_cast(KV.first); + dbgs() << " " << Prot << ": " + << (void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + }); + + // Update atom target addresses. + for (auto &KV : SegAtoms) { + auto &Prot = KV.first; + auto &Atoms = KV.second; + + JITTargetAddress AtomTargetAddr = + Alloc->getTargetMemory(static_cast(Prot)); + + for (auto *A : Atoms) { + AtomTargetAddr = alignTo(AtomTargetAddr, A->getAlignment()); + A->setAddress(AtomTargetAddr); + AtomTargetAddr += A->getContent().size(); + } + } + + return Error::success(); +} + +DenseSet JITLinkerBase::identifyUnresolvedExternals() const { + // Identify unresolved external atoms. + DenseSet UnresolvedExternals; + for (auto *DA : G->external_atoms()) + if (!DA->isResolved()) { + assert(DA->getName() != StringRef() && DA->getName() != "" && + "Externals must be named"); + UnresolvedExternals.insert(DA->getName()); + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto &KV : Result) { + Atom &A = G->getAtomByName(KV.first); + assert(!A.isResolved() && "Atom already resolved"); + A.setAddress(KV.second.getAddress()); + } + + assert(llvm::all_of(G->external_atoms(), + [](Atom *A) { return A->isResolved(); }) && + "All atoms should have been resolved by this point"); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void deadStrip(AtomGraph &G) { + std::vector Worklist; + + // Build the initial worklist from all atoms initially live. + for (auto *DA : G.defined_atoms()) { + if (!DA->isLive()) + continue; + + for (auto &E : DA->edges()) + if (E.isKeepAlive() && E.getTarget().isDefined() && + !E.getTarget().isLive()) + Worklist.push_back(static_cast(&E.getTarget())); + } + + // Propagate live flags to all atoms reachable from the initial live set. + while (!Worklist.empty()) { + DefinedAtom &NextLive = *Worklist.back(); + Worklist.pop_back(); + + // If this atom has already been marked as live then skip it. + if (NextLive.isLive()) + continue; + + // Otherwise set it as live and add any non-live atoms that it points to + // to the worklist. + NextLive.setIsLive(true); + + for (auto &E : NextLive.edges()) + if (E.isKeepAlive() && E.getTarget().isDefined() && + !E.getTarget().isLive()) + Worklist.push_back(static_cast(&E.getTarget())); + } + + // Collect atoms to remove, then remove them from the graph. + std::vector AtomsToRemove; + for (auto *DA : G.defined_atoms()) + if (!DA->isLive()) + AtomsToRemove.push_back(DA); + + LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); + for (auto *DA : AtomsToRemove) { + LLVM_DEBUG(dbgs() << " " << *DA << "\n"); + G.removeAtom(*DA); + } +} + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) { + return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, + FDEToCIERelocKind, FDEToTargetRelocKind) + .atomize(); +} + +} // end namespace jitlink +} // end namespace llvm Index: lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp @@ -0,0 +1,627 @@ +//===------- JITLink_MachO_x86_64.cpp - JIT linker functionality ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } +} + +class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +public: + MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOAtomGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { + return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), + EHFrameSection.getContent(), + EHFrameSection.getAddress(), NegDelta32, Pointer64); + }); + } + +private: + static Expected + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel && RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + if (!RI.r_pcrel && RI.r_extern) + // SUBTRACTOR length must be 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta'. They may + // be turned into NegDelta by parsePairRelocation. + if (RI.r_length == 2) + return Delta32; + if (RI.r_length == 3) + return Delta64; + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error("Unsupported x86-64 relocation kind"); + } + + Expected findAtomBySymbolIndex(const MachO::relocation_info &RI) { + auto &Obj = getObject(); + if (RI.r_symbolnum >= NumSymbols) + return make_error("Symbol index out of range"); + auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); + auto Name = SymI->getName(); + if (!Name) + return Name.takeError(); + return getGraph().getAtomByName(*Name); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = std::tuple; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected + parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + auto FromAtom = findAtomBySymbolIndex(SubRI); + if (!FromAtom) + return FromAtom.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const ulittle64_t *)FixupContent; + else + FixupValue = *(const ulittle32_t *)FixupContent; + + // Find 'ToAtom' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + if (UnsignedRI.r_extern) { + // Find the target atom. + auto ToAtom = findAtomBySymbolIndex(UnsignedRI); + if (!ToAtom) + return ToAtom.takeError(); + + // Use Delta32/64 or NegDelta32/64 depending on whether it's the "To" + // atom or the "From" atom that is being fixed up. + if (&AtomToFix == &*FromAtom) + return PairRelocInfo{SubRI.r_length == 3 ? Delta64 : Delta32, &*ToAtom, + FixupValue + + (FixupAddress - FromAtom->getAddress())}; + else if (&AtomToFix == &*ToAtom) + return PairRelocInfo{ + SubRI.r_length == 3 ? NegDelta64 : NegDelta32, &*FromAtom, + FixupValue - (FixupAddress - ToAtom->getAddress())}; + + // AtomToFix was neither FromAtom nor ToAtom. + return make_error("SUBTRACTOR relocation must fix up " + "either 'A' or 'B'"); + } + + // If we get here we have a non-extern UNSIGNED. + if (&AtomToFix != &*FromAtom) + return make_error("Non-extern delta must be based on " + "fixup atom"); + + JITTargetAddress TargetAddr = FixupAddress + FixupValue; + auto ToAtom = getGraph().findAtomByAddress(TargetAddr); + if (!ToAtom) + return ToAtom.takeError(); + + return PairRelocInfo{SubRI.r_length == 3 ? Delta64 : Delta32, &*ToAtom, + TargetAddr - ToAtom->getAddress()}; + } + + Error addRelocations() override { + using namespace support; + auto &G = getGraph(); + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG(dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"); + + // Find the atom that the fixup points to. + DefinedAtom *AtomToFix = nullptr; + { + auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); + if (!AtomToFixOrErr) + return AtomToFixOrErr.takeError(); + AtomToFix = &*AtomToFixOrErr; + } + + if (FixupAddress + (1 << RI.r_length) > + AtomToFix->getAddress() + AtomToFix->getContent().size()) + return make_error( + "Relocation content extends past end of fixup atom"); + + // Get a pointer to the fixup content. + const char *FixupContent = AtomToFix->getContent().data() + + (FixupAddress - AtomToFix->getAddress()); + + // The target atom and addend will be populated by the switch below. + Atom *TargetAtom = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = 1 << (*Kind - PCRel32Minus1Anon); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetAtom, Addend) = *PairInfo; + assert(TargetAtom && "No target atom from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG(Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + printEdge(dbgs(), *AtomToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n";); + AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachOInPlaceGOTAndStubsBuilder { +public: + MachOInPlaceGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + + void run() { + // We're going to be adding new atoms, but we don't want to iterate over + // the newly added ones, so just copy the existing atoms out. + std::vector DAs(G.defined_atoms().begin(), + G.defined_atoms().end()); + + for (auto *DA : DAs) + for (auto &E : DA->edges()) + if (E.getKind() == PCRel32GOTLoad) + fixGOTEdge(E); + else if (E.getKind() == Branch32 && !E.getTarget().isDefined()) + fixExternalBranchEdge(E); + } + + Atom &getGOTEntryAtom(Atom &Target) { + assert(!Target.getName().empty() && + "GOT load edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + // Build a GOT section if we don't have one already. + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ, 8); + + auto &GOTEntryAtom = G.addAnonymousAtom(*GOTSection, 0x0, 8); + GOTEntryAtom.setContent( + StringRef(reinterpret_cast(NullGOTEntryContent), 8)); + GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntryAtom)) + .first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + return *GOTEntryI->second; + } + + void fixGOTEdge(Edge &E) { + assert(E.getKind() == PCRel32GOTLoad && "Not a GOT edge?"); + assert(E.getAddend() == 0 && "GOT edge has non-zero addend?"); + auto &GOTEntryAtom = getGOTEntryAtom(E.getTarget()); + E.setKind(PCRel32); + E.setTarget(GOTEntryAtom); + } + + Atom &getStubAtom(Atom &Target) { + assert(!Target.getName().empty() && + "Branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + // Build a Stubs section if we don't have one already. + if (!StubsSection) { + auto StubsProt = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt, 8); + } + + auto &StubAtom = G.addAnonymousAtom(*StubsSection, 0x0, 2); + StubAtom.setContent( + StringRef(reinterpret_cast(StubContent), 6)); + + // Re-use GOT entries for stub targets. + auto &GOTEntryAtom = getGOTEntryAtom(Target); + StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); + + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub atom"); + return *StubI->second; + } + + void fixExternalBranchEdge(Edge &E) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(getStubAtom(E.getTarget())); + } + + AtomGraph &G; + DenseMap GOTEntries; + DenseMap Stubs; + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachOInPlaceGOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachOInPlaceGOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker { + friend class JITLinker; + +public: + MachOJITLinker_x86_64(std::unique_ptr ObjBuffer, + JITLinkMemoryManager &MemMgr, + JITLinkAsyncLookupFunction Lookup, + OnResolveFunction OnResolved, + OnFinalizeFunction OnFinalized, + AtomGraphPassFunction MarkLivePass, + AtomGraphPassFunction GOTAndStubsPass) + : JITLinker(std::move(ObjBuffer), MemMgr, std::move(Lookup), + std::move(OnResolved), std::move(OnFinalized)), + MarkLivePass(std::move(MarkLivePass)), + GOTAndStubsPass(std::move(GOTAndStubsPass)) { + assert(this->MarkLivePass && "MarkLivePass should be set"); + assert(this->GOTAndStubsPass && "GOTAndStubsPass should be set"); + } + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + Expected> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Target \"" << E.getTarget() << " out of range"; + } + return make_error(std::move(ErrMsg)); + } + + Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + using namespace support; + + char *FixupPtr = AtomWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case PCRel32: + case PCRel32Anon: + case Branch32: { + LLVM_DEBUG(dbgs() << "Applying PCRel32/PCRel32Anon/Branch to " << A + << ": Target addr = " + << format("0x%016" PRIx64, E.getTarget().getAddress()) + << " (" << E.getTarget() << ")" + << ", Fixup addr = " + << format("0x%016" PRIx64, FixupAddress) + << ", Addend = " << E.getAddend() << "\n";); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus1Anon: { + // FIXME: Implement. + break; + } + case PCRel32Minus2: + case PCRel32Minus2Anon: { + // FIXME: Implement. + break; + } + case PCRel32Minus4: + case PCRel32Minus4Anon: { + // FIXME: Implement. + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + Error runPreDeadStripPasses(AtomGraph &G) override { return MarkLivePass(G); } + + Error runPostDeadStripPasses(AtomGraph &G) override { + return GOTAndStubsPass(G); + } + + uint64_t NullValue = 0; + AtomGraphPassFunction MarkLivePass; + AtomGraphPassFunction GOTAndStubsPass; +}; + +void jitLink_MachO_x86_64(std::unique_ptr ObjBuffer, + JITLinkMemoryManager &MemMgr, + JITLinkAsyncLookupFunction Lookup, + OnResolveFunction OnResolved, + OnFinalizeFunction OnFinalized, + AtomGraphPassFunction MarkLivePass, + AtomGraphPassFunction GOTAndStubsPass) { + + if (!MarkLivePass) + MarkLivePass = markAllAtomsLive; + + if (!GOTAndStubsPass) + GOTAndStubsPass = [](AtomGraph &G) -> Error { + MachOInPlaceGOTAndStubsBuilder(G).run(); + return Error::success(); + }; + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(ObjBuffer), MemMgr, std::move(Lookup), + std::move(OnResolved), std::move(OnFinalized), + std::move(MarkLivePass), + std::move(GOTAndStubsPass)); +} + +} // end namespace jitlink +} // end namespace llvm Index: lib/ExecutionEngine/JITLink/LLVMBuild.txt =================================================================== --- lib/ExecutionEngine/JITLink/LLVMBuild.txt +++ lib/ExecutionEngine/JITLink/LLVMBuild.txt @@ -1,4 +1,4 @@ -;===- ./lib/ExecutionEngine/LLVMBuild.txt ----------------------*- Conf -*--===; +;===----- ./lib/ExecutionEngine/JTILink/LLVMBuild.txt ----------*- Conf -*--===; ; ; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ; See https://llvm.org/LICENSE.txt for license information. @@ -14,11 +14,8 @@ ; ;===------------------------------------------------------------------------===; -[common] -subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT Orc PerfJITEvents - [component_0] type = Library -name = ExecutionEngine -parent = Libraries -required_libraries = Core MC Object RuntimeDyld Support Target +name = JITLink +parent = ExecutionEngine +required_libraries = Object Support Index: lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h @@ -0,0 +1,86 @@ +//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "JITLinkGeneric.h" + +#include "llvm/Object/MachO.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace jitlink { + +class MachOAtomGraphBuilder { +public: + virtual ~MachOAtomGraphBuilder(); + Expected> buildGraph(); + +protected: + using OffsetToAtomMap = std::map; + + class MachOSection { + public: + MachOSection() = default; + MachOSection(Section &GenericSection, StringRef Content, + JITTargetAddress Address) + : GenericSection(&GenericSection), Content(Content), Address(Address) {} + Section &getGenericSection() const { + assert(GenericSection && "Section is null"); + return *GenericSection; + } + StringRef getName() const { + assert(GenericSection && "No generic section attached"); + return GenericSection->getName(); + } + StringRef getContent() const { return Content; } + JITTargetAddress getAddress() const { return Address; } + + private: + Section *GenericSection = nullptr; + StringRef Content; + JITTargetAddress Address = 0; + }; + + using CustomAtomizeFunction = std::function; + + MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); + + AtomGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); + + virtual Error addRelocations() = 0; + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + Error parseSections(); + Error addNonCustomAtoms(); + Error addAtoms(); + + const object::MachOObjectFile &Obj; + std::unique_ptr G; + DenseMap Sections; + StringMap CustomAtomizeFunctions; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H Index: lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp @@ -0,0 +1,229 @@ +//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} + +Expected> MachOAtomGraphBuilder::buildGraph() { + if (auto Err = parseSections()) + return std::move(Err); + + if (auto Err = addAtoms()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(llvm::make_unique(getPointerSize(Obj), getEndianness(Obj))) { +} + +void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, + CustomAtomizeFunction Atomizer) { + assert(!CustomAtomizeFunctions.count(SectionName) && + "Custom atomizer for this section already exists"); + CustomAtomizeFunctions[SectionName] = std::move(Atomizer); +} + +unsigned +MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +Error MachOAtomGraphBuilder::parseSections() { + for (auto &SecRef : Obj.sections()) { + assert((SecRef.getAlignment() <= std::numeric_limits::max()) && + "Section alignment does not fit in 32 bits"); + + StringRef Name; + if (auto EC = SecRef.getName(Name)) + return errorCodeToError(EC); + + StringRef Content; + if (auto EC = SecRef.getContents(Content)) + return errorCodeToError(EC); + + unsigned SectionIndex = SecRef.getIndex() + 1; + + LLVM_DEBUG(dbgs() << "Adding section " << Name << ": " + << format("0x%016" PRIx64, SecRef.getAddress()) + << ", size: " << Content.size() + << ", align: " << SecRef.getAlignment() << "\n";); + + // FIXME: Get real section permissions + // How, exactly, on MachO? + sys::Memory::ProtectionFlags Prot; + if (SecRef.isText()) + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto &GenericSection = G->createSection(Name, Prot, SecRef.getAlignment()); + Sections[SectionIndex] = + MachOSection(GenericSection, Content, SecRef.getAddress()); + } + + return Error::success(); +} + +// Adds atoms with identified start addresses (but not lengths) for all named +// atoms. +// Also, for every section that contains named atoms, but does not have an +// atom at offset zero of that section, constructs an anonymous atom covering +// that range. +Error MachOAtomGraphBuilder::addNonCustomAtoms() { + using AddrToAtomMap = std::map; + DenseMap SecToAtoms; + + for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; + ++SymI) { + object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + + auto Addr = Sym.getAddress(); + if (!Addr) + return Addr.takeError(); + + uint32_t Flags = Sym.getFlags(); + + if (Flags & object::SymbolRef::SF_Undefined) { + LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); + G->addExternalAtom(*Name); + continue; + } else if (Flags & object::SymbolRef::SF_Absolute) { + LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " + << format("0x%016" PRIx64, *Addr) << "\n"); + G->addAbsoluteAtom(*Name, *Addr); + continue; + } + + LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); + + // This atom is neither undefined nor absolute, so it must be defined in + // this object. Get its section index. + auto SecItr = Sym.getSection(); + if (!SecItr) + return SecItr.takeError(); + + uint64_t SectionIndex = (*SecItr)->getIndex() + 1; + + auto SecByIndexItr = Sections.find(SectionIndex); + if (SecByIndexItr == Sections.end()) + return make_error("Unrecognized section index in macho"); + + auto &Sec = SecByIndexItr->second; + auto &SecAtoms = SecToAtoms[&Sec]; + auto &A = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U)); + + LLVM_DEBUG(dbgs() << "Added " << *Name + << " addr: " << format("0x%016" PRIx64, *Addr) + << ", align: " << A.getAlignment() << ", section: " + << Sec.getGenericSection().getName() << "\n";); + + SecAtoms[A.getAddress() - Sec.getAddress()] = &A; + } + + // Add anonymous atoms. + for (auto &KV : Sections) { + auto &S = KV.second; + + // Skip empty sections. + if (S.getContent().empty()) + continue; + + // Skip sections with custom handling. + if (CustomAtomizeFunctions.count(S.getName())) + continue; + + auto SAI = SecToAtoms.find(&S); + + // If S is not in the SecToAtoms map then it contained no named atom. Add + // one anonymous atom to cover the whole section. + if (SAI == SecToAtoms.end()) { + SecToAtoms[&S][0] = + &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getGenericSection().getAlignment()); + continue; + } + + // Otherwise, check whether this section had an atom covering offset zero. + // If not, add one. + auto &SecAtoms = SAI->second; + if (!SecAtoms.count(0)) + SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getGenericSection().getAlignment()); + } + + LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); + + // Set atom contents. + for (auto &KV : SecToAtoms) { + auto &S = *KV.first; + auto &SecAtoms = KV.second; + + // Iterate the atoms in reverse order and set up their contents. + JITTargetAddress LastAtomAddr = S.getContent().size(); + for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { + auto Offset = I->first; + auto &A = *I->second; + LLVM_DEBUG(dbgs() << " " << A << " to [ " << S.getAddress() + Offset + << " .. " << S.getAddress() + LastAtomAddr << " ]\n";); + A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); + LastAtomAddr = Offset; + } + } + + return Error::success(); +} + +Error MachOAtomGraphBuilder::addAtoms() { + // Add all named atoms. + if (auto Err = addNonCustomAtoms()) + return Err; + + // Process special sections. + for (auto &KV : Sections) { + auto &S = KV.second; + auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); + if (HI != CustomAtomizeFunctions.end()) { + auto &Atomize = HI->second; + if (auto Err = Atomize(S)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm Index: lib/ExecutionEngine/LLVMBuild.txt =================================================================== --- lib/ExecutionEngine/LLVMBuild.txt +++ lib/ExecutionEngine/LLVMBuild.txt @@ -15,7 +15,8 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT Orc PerfJITEvents +subdirectories = Interpreter MCJIT JITLink RuntimeDyld IntelJITEvents + OProfileJIT Orc PerfJITEvents [component_0] type = Library Index: lib/Support/Memory.cpp =================================================================== --- lib/Support/Memory.cpp +++ lib/Support/Memory.cpp @@ -15,6 +15,10 @@ #include "llvm/Config/llvm-config.h" #include "llvm/Support/Valgrind.h" +#ifndef NDEBUG +#include "llvm/Support/raw_ostream.h" +#endif // ifndef NDEBUG + // Include the platform-specific parts of this class. #ifdef LLVM_ON_UNIX #include "Unix/Memory.inc" @@ -22,3 +26,28 @@ #ifdef _WIN32 #include "Windows/Memory.inc" #endif + +#ifndef NDEBUG + +namespace llvm { +namespace sys { + +raw_ostream &operator<<(raw_ostream &OS, const Memory::ProtectionFlags &PF) { + assert((PF & ~(Memory::MF_READ | Memory::MF_WRITE | Memory::MF_EXEC)) == 0 && + "Unrecognized flags"); + + return OS << (PF & Memory::MF_READ ? 'R' : '-') + << (PF & Memory::MF_WRITE ? 'W' : '-') + << (PF & Memory::MF_EXEC ? 'X' : '-'); +} + +raw_ostream &operator<<(raw_ostream &OS, const MemoryBlock &MB) { + return OS << "[ " << MB.base() << " .. " + << (void *)((char *)MB.base() + MB.size()) << " ] (" << MB.size() + << " bytes)"; +} + +} // end namespace sys +} // end namespace llvm + +#endif // ifndef NDEBUG Index: unittests/ExecutionEngine/CMakeLists.txt =================================================================== --- unittests/ExecutionEngine/CMakeLists.txt +++ unittests/ExecutionEngine/CMakeLists.txt @@ -12,6 +12,7 @@ ExecutionEngineTest.cpp ) +add_subdirectory(JITLink) add_subdirectory(Orc) # Include MCJIT tests only if native arch is a built JIT target. Index: unittests/ExecutionEngine/JITLink/CMakeLists.txt =================================================================== --- /dev/null +++ unittests/ExecutionEngine/JITLink/CMakeLists.txt @@ -0,0 +1,15 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + JITLink + MC + Object + RuntimeDyld + Support + ) + +add_llvm_unittest(JITLinkTests + JITLinkTestCommon.cpp + JITLinkTest_MachO_x86_64_Tests.cpp + ) + +target_link_libraries(JITLinkTests PRIVATE LLVMTestingSupport) Index: unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h =================================================================== --- /dev/null +++ unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h @@ -0,0 +1,154 @@ +//===---- JITLinkTestCommon.h - Utilities for Orc Unit Tests ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Common utilities for JITLink unit tests. +// +//===----------------------------------------------------------------------===// + + +#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTCOMMON_H +#define LLVM_UNITTESTS_EXECUTIONENGINE_JITLINK_JITLINKTESTCOMMON_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" + +#include "gtest/gtest.h" + +namespace llvm { + +class JITLinkTestCommon { +public: + + class TestResources { + public: + TestResources(StringRef AsmSrc, StringRef TripleStr, + StringMap Externals, bool PIC, + bool LargeCodeModel, MCTargetOptions Options); + + std::unique_ptr getTestObjectBuffer() const; + + JITTargetAddress getExternalAddress(StringRef Name) const; + + jitlink::JITLinkAsyncLookupFunction getExternalsResolver(); + + const MCDisassembler &getDisassembler() const { return *Dis; } + + private: + void initializeTripleSpecifics(Triple &TT); + void initializeTestSpecifics(StringRef AsmSource, const Triple &TT, + bool PIC, bool LargeCodeModel); + + const Target *TheTarget = nullptr; + SourceMgr SrcMgr; + SmallVector ObjBuffer; + raw_svector_ostream ObjStream; + StringMap Externals; + + MCTargetOptions Options; + std::unique_ptr MRI; + std::unique_ptr MAI; + std::unique_ptr MCII; + std::unique_ptr STI; + + MCObjectFileInfo MOFI; + std::unique_ptr AsCtx; + std::unique_ptr MOS; + + std::unique_ptr DisCtx; + std::unique_ptr Dis; + }; + + JITLinkTestCommon(); + + std::unique_ptr + getTestResources(StringRef AsmSrc, StringRef Triple, + StringMap Externals, bool PIC, + bool LargeCodeModel, MCTargetOptions Options) const { + return llvm::make_unique(AsmSrc, Triple, + std::move(Externals), PIC, + LargeCodeModel, std::move(Options)); + } + + template + static Expected readInt(jitlink::AtomGraph &G, jitlink::DefinedAtom &A, + size_t Offset = 0) { + if (Offset + sizeof(T) > A.getContent().size()) + return make_error("Reading past end of atom content", + inconvertibleErrorCode()); + return support::endian::read(A.getContent().data() + Offset, + G.getEndianness()); + } + + template + static Expected readInt(jitlink::AtomGraph &G, StringRef AtomName, + size_t Offset = 0) { + auto DA = G.findDefinedAtomByName(AtomName); + if (!DA) + return DA.takeError(); + return readInt(G, *DA); + } + + static Expected> + disassemble(const MCDisassembler &Dis, jitlink::DefinedAtom &Atom, + size_t Offset = 0); + + static Expected decodeImmediateOperand(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, + size_t OpIdx, + size_t Offset = 0); + + static jitlink::Atom &atom(jitlink::AtomGraph &G, StringRef Name) { + return G.getAtomByName(Name); + } + + static jitlink::DefinedAtom &definedAtom(jitlink::AtomGraph &G, + StringRef Name) { + return G.getDefinedAtomByName(Name); + } + + static JITTargetAddress atomAddr(jitlink::AtomGraph &G, StringRef Name) { + return atom(G, Name).getAddress(); + } + + template + static size_t countEdgesMatching(jitlink::DefinedAtom &DA, + const PredT &Pred) { + return std::count_if(DA.edges().begin(), DA.edges().end(), Pred); + } + + template + static size_t countEdgesMatching(jitlink::AtomGraph &G, StringRef Name, + const PredT &Pred) { + return countEdgesMatching(definedAtom(G, Name), Pred); + } + +private: + + static bool AreTargetsInitialized; + void initializeLLVMTargets(); + + DenseMap Externals; +}; + +} // end namespace llvm + +#endif Index: unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp =================================================================== --- /dev/null +++ unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp @@ -0,0 +1,174 @@ +//===------- JITLinkTestCommon.cpp - Common code for JITLink tests --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "JITLinkTestCommon.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/Support/TargetSelect.h" + +namespace llvm { + +JITLinkTestCommon::TestResources::TestResources( + StringRef AsmSrc, StringRef TripleStr, + StringMap Externals, bool PIC, bool LargeCodeModel, + MCTargetOptions Options) + : ObjStream(ObjBuffer), Externals(std::move(Externals)), + Options(std::move(Options)) { + Triple TT(Triple::normalize(TripleStr)); + initializeTripleSpecifics(TT); + initializeTestSpecifics(AsmSrc, TT, PIC, LargeCodeModel); +} + +std::unique_ptr +JITLinkTestCommon::TestResources::getTestObjectBuffer() const { + return MemoryBuffer::getMemBuffer( + StringRef(ObjBuffer.data(), ObjBuffer.size()), "Test object buffer", + false); +} + +JITTargetAddress +JITLinkTestCommon::TestResources::getExternalAddress(StringRef Name) const { + auto I = Externals.find(Name); + assert(I != Externals.end() && "No external with the given name"); + return I->second.getAddress(); +} + +jitlink::JITLinkAsyncLookupFunction +JITLinkTestCommon::TestResources::getExternalsResolver() { + return [this](const DenseSet &Symbols, + jitlink::JITLinkAsyncLookupContinuation LookupContinuation) { + jitlink::AsyncLookupResult LookupResult; + for (const auto &Symbol : Symbols) + LookupResult[Symbol] = Externals[Symbol]; + LookupContinuation(std::move(LookupResult)); + }; +} + +void JITLinkTestCommon::TestResources::initializeTripleSpecifics(Triple &TT) { + std::string Error; + TheTarget = TargetRegistry::lookupTarget("", TT, Error); + + if (!TheTarget) + report_fatal_error(Error); + + MRI.reset(TheTarget->createMCRegInfo(TT.getTriple())); + if (!MRI) + report_fatal_error("Could not build MCRegisterInfo for triple"); + + MAI.reset(TheTarget->createMCAsmInfo(*MRI, TT.getTriple())); + if (!MAI) + report_fatal_error("Could not build MCAsmInfo for triple"); + + MCII.reset(TheTarget->createMCInstrInfo()); + if (!MCII) + report_fatal_error("Could not build MCInstrInfo for triple"); + + STI.reset(TheTarget->createMCSubtargetInfo(TT.getTriple(), "", "")); + if (!STI) + report_fatal_error("Could not build MCSubtargetInfo for triple"); + + DisCtx = llvm::make_unique(MAI.get(), MRI.get(), nullptr); + Dis.reset(TheTarget->createMCDisassembler(*STI, *DisCtx)); + + if (!Dis) + report_fatal_error("Could not build MCDisassembler"); +} + +void JITLinkTestCommon::TestResources::initializeTestSpecifics( + StringRef AsmSrc, const Triple &TT, bool PIC, bool LargeCodeModel) { + SrcMgr.AddNewSourceBuffer(MemoryBuffer::getMemBuffer(AsmSrc), SMLoc()); + AsCtx = llvm::make_unique(MAI.get(), MRI.get(), &MOFI, &SrcMgr); + MOFI.InitMCObjectFileInfo(TT, PIC, *AsCtx, LargeCodeModel); + + std::unique_ptr CE( + TheTarget->createMCCodeEmitter(*MCII, *MRI, *AsCtx)); + if (!CE) + report_fatal_error("Could not build MCCodeEmitter"); + + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, Options)); + if (!MAB) + report_fatal_error("Could not build MCAsmBackend for test"); + + std::unique_ptr MOW(MAB->createObjectWriter(ObjStream)); + + MOS.reset(TheTarget->createMCObjectStreamer( + TT, *AsCtx, std::move(MAB), std::move(MOW), std::move(CE), *STI, + Options.MCRelaxAll, Options.MCIncrementalLinkerCompatible, false)); + + std::unique_ptr MAP( + createMCAsmParser(SrcMgr, *AsCtx, *MOS, *MAI)); + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *MAP, *MCII, Options)); + + if (!TAP) + report_fatal_error("Could not build MCTargetAsmParser for test"); + + MAP->setTargetParser(*TAP); + + if (MAP->Run(false)) + report_fatal_error("Failed to parse test case"); +} + +JITLinkTestCommon::JITLinkTestCommon() { initializeLLVMTargets(); } + +Expected> +JITLinkTestCommon::disassemble(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, size_t Offset) { + ArrayRef InstBuffer( + reinterpret_cast(Atom.getContent().data()) + Offset, + Atom.getContent().size() - Offset); + + MCInst Inst; + uint64_t InstSize; + auto Status = + Dis.getInstruction(Inst, InstSize, InstBuffer, 0, nulls(), nulls()); + + if (Status != MCDisassembler::Success) + return make_error("Could not disassemble instruction", + inconvertibleErrorCode()); + + return std::make_pair(Inst, InstSize); +} + +Expected +JITLinkTestCommon::decodeImmediateOperand(const MCDisassembler &Dis, + jitlink::DefinedAtom &Atom, + size_t OpIdx, size_t Offset) { + auto InstAndSize = disassemble(Dis, Atom, Offset); + if (!InstAndSize) + return InstAndSize.takeError(); + + if (OpIdx >= InstAndSize->first.getNumOperands()) + return make_error("Invalid operand index", + inconvertibleErrorCode()); + + auto &Op = InstAndSize->first.getOperand(OpIdx); + + if (!Op.isImm()) + return make_error("Operand at index is not immediate", + inconvertibleErrorCode()); + + return Op.getImm(); +} + +bool JITLinkTestCommon::AreTargetsInitialized = false; + +void JITLinkTestCommon::initializeLLVMTargets() { + if (!AreTargetsInitialized) { + InitializeAllTargets(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + InitializeAllAsmPrinters(); + InitializeAllDisassemblers(); + AreTargetsInitialized = true; + } +} + +} // end namespace llvm Index: unittests/ExecutionEngine/JITLink/JITLinkTest_MachO_x86_64_Tests.cpp =================================================================== --- /dev/null +++ unittests/ExecutionEngine/JITLink/JITLinkTest_MachO_x86_64_Tests.cpp @@ -0,0 +1,229 @@ +//===---- JITLinkTest_MachO_x86_64.cpp - Tests for JITLink MachO/x86-64 ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "JITLinkTestCommon.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" +#include "llvm/Testing/Support/Error.h" + +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class JITLinkTest_MachO_x86_64 : public JITLinkTestCommon, + public testing::Test { +public: + using BasicVerifyGraphFunction = + std::function; + + void runBasicVerifyGraphTest(StringRef AsmSrc, StringRef Triple, + StringMap Externals, + bool PIC, bool LargeCodeModel, + MCTargetOptions Options, + BasicVerifyGraphFunction RunGraphTest) { + auto TR = getTestResources(AsmSrc, Triple, std::move(Externals), PIC, + LargeCodeModel, std::move(Options)); + + auto MemMgr = make_unique(); + + jitLink_MachO_x86_64( + TR->getTestObjectBuffer(), *MemMgr, TR->getExternalsResolver(), + [&](Expected G) { + if (G) + RunGraphTest(*G, TR->getDisassembler()); + else + ADD_FAILURE() << "Unexpected linker failure"; + }, + [](Expected> Alloc) { + EXPECT_THAT_EXPECTED(std::move(Alloc), Succeeded()); + }); + } + +protected: + static void verifyIsPointerTo(AtomGraph &G, DefinedAtom &A, Atom &Target) { + EXPECT_EQ(A.edges_size(), 1U) << "Incorrect number of edges for pointer"; + if (A.edges_size() != 1U) + return; + auto &E = *A.edges().begin(); + EXPECT_EQ(E.getKind(), Pointer64) + << "Expected pointer to have a pointer64 relocation"; + EXPECT_EQ(&E.getTarget(), &Target) << "Expected edge to point at target"; + EXPECT_THAT_EXPECTED(readInt(G, A), HasValue(Target.getAddress())) + << "Pointer does not point to target"; + } + + static void verifyGOTLoad(AtomGraph &G, DefinedAtom &A, Edge &E, + Atom &Target) { + EXPECT_EQ(E.getAddend(), 0U) << "Expected GOT load to have a zero addend"; + EXPECT_TRUE(E.getTarget().isDefined()) + << "GOT entry should be a defined atom"; + if (!E.getTarget().isDefined()) + return; + + verifyIsPointerTo(G, static_cast(E.getTarget()), Target); + } + + static void verifyCall(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + EXPECT_EQ(E.getKind(), Branch32) << "Edge is not a Branch32"; + EXPECT_EQ(E.getAddend(), 0U) << "Expected no addend on stub call"; + EXPECT_EQ(&E.getTarget(), &Callee) + << "Edge does not point at expected callee"; + + JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + uint64_t PCRelDelta = Callee.getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED( + decodeImmediateOperand(Dis, Caller, 0, E.getOffset() - 1), + HasValue(PCRelDelta)); + } + + static void verifyIndirectCall(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + EXPECT_EQ(E.getKind(), PCRel32) << "Edge is not a PCRel32"; + EXPECT_EQ(E.getAddend(), 0) << "Expected no addend on stub cal"; + EXPECT_TRUE(E.getTarget().isDefined()) << "Target is not a defined atom"; + if (!E.getTarget().isDefined()) + return; + verifyIsPointerTo(G, static_cast(E.getTarget()), Callee); + + JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + uint64_t PCRelDelta = E.getTarget().getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED( + decodeImmediateOperand(Dis, Caller, 3, E.getOffset() - 2), + HasValue(PCRelDelta)); + } + + static void verifyCallViaStub(const MCDisassembler &Dis, AtomGraph &G, + DefinedAtom &Caller, Edge &E, Atom &Callee) { + verifyCall(Dis, G, Caller, E, E.getTarget()); + + if (!E.getTarget().isDefined()) { + ADD_FAILURE() << "Edge target is not a stub"; + return; + } + + auto &StubAtom = static_cast(E.getTarget()); + EXPECT_EQ(StubAtom.edges_size(), 1U) + << "Expected one edge from stub to target"; + + auto &StubEdge = *StubAtom.edges().begin(); + + verifyIndirectCall(Dis, G, static_cast(StubAtom), StubEdge, + Callee); + } +}; + +} // end anonymous namespace + +// Test each operation on LegacyObjectTransformLayer. +TEST_F(JITLinkTest_MachO_x86_64, BasicRelocations) { + runBasicVerifyGraphTest( + R"( + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 10, 14 + .globl _bar + .p2align 4, 0x90 + _bar: + callq _baz + + .globl _foo + .p2align 4, 0x90 + _foo: + callq _bar + _foo.1: + movq _y@GOTPCREL(%rip), %rcx + _foo.2: + movq _p(%rip), %rdx + + .section __DATA,__data + .globl _x + .p2align 2 + _x: + .long 42 + + .globl _p + .p2align 3 + _p: + .quad _x + + .subsections_via_symbols)", + "x86_64-apple-macosx10.14", + {{"_y", JITEvaluatedSymbol(0xdeadbeef, JITSymbolFlags::Exported)}}, true, + false, MCTargetOptions(), [](AtomGraph &G, const MCDisassembler &Dis) { + // Name the atoms in the asm above. + auto &Baz = atom(G, "_baz"); + auto &Y = atom(G, "_y"); + + auto &Bar = definedAtom(G, "_bar"); + auto &Foo = definedAtom(G, "_foo"); + auto &Foo_1 = definedAtom(G, "_foo.1"); + auto &Foo_2 = definedAtom(G, "_foo.2"); + auto &X = definedAtom(G, "_x"); + auto &P = definedAtom(G, "_p"); + + // Check unsigned reloc for _p + { + EXPECT_EQ(P.edges_size(), 1U) << "Unexpected number of relocations"; + EXPECT_EQ(P.edges().begin()->getKind(), Pointer64) + << "Unexpected edge kind for _p"; + EXPECT_THAT_EXPECTED(readInt(G, P), + HasValue(X.getAddress())) + << "Unsigned relocation did not apply correctly"; + } + + // Check that _bar is a call-via-stub to _baz. + // This will check that the call goes to a stub, that the stub is an + // indirect call, and that the pointer for the indirect call points to + // baz. + { + EXPECT_EQ(Bar.edges_size(), 1U) + << "Incorrect number of edges for bar"; + EXPECT_EQ(Bar.edges().begin()->getKind(), Branch32) + << "Unexpected edge kind for _bar"; + verifyCallViaStub(Dis, G, Bar, *Bar.edges().begin(), Baz); + } + + // Check that _foo is a direct call to _bar. + { + EXPECT_EQ(Foo.edges_size(), 1U) + << "Incorrect number of edges for foo"; + EXPECT_EQ(Foo.edges().begin()->getKind(), Branch32); + verifyCall(Dis, G, Foo, *Foo.edges().begin(), Bar); + } + + // Check .got load in _foo.1 + { + EXPECT_EQ(Foo_1.edges_size(), 1U) + << "Incorrect number of edges for foo_1"; + EXPECT_EQ(Foo_1.edges().begin()->getKind(), PCRel32); + verifyGOTLoad(G, Foo_1, *Foo_1.edges().begin(), Y); + } + + // Check PCRel ref to _p in _foo.2 + { + EXPECT_EQ(Foo_2.edges_size(), 1U) + << "Incorrect number of edges for foo_2"; + EXPECT_EQ(Foo_2.edges().begin()->getKind(), PCRel32); + + JITTargetAddress FixupAddress = + Foo_2.getAddress() + Foo_2.edges().begin()->getOffset(); + uint64_t PCRelDelta = P.getAddress() - (FixupAddress + 4); + + EXPECT_THAT_EXPECTED(decodeImmediateOperand(Dis, Foo_2, 4, 0), + HasValue(PCRelDelta)) + << "PCRel load does not reference expected target"; + } + }); +}