diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h @@ -57,7 +57,7 @@ std::unique_ptr Ctx); /// Return the string name of the given ELF x86-64 edge kind. -StringRef getELFX86RelocationKindName(Edge::Kind R); +const char *getELFX86RelocationKindName(Edge::Kind R); } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -792,10 +792,13 @@ Section::const_block_iterator, const Block *, getSectionConstBlocks>; + using GetEdgeKindNameFunction = const char *(*)(Edge::Kind); + LinkGraph(std::string Name, const Triple &TT, unsigned PointerSize, - support::endianness Endianness) + support::endianness Endianness, + GetEdgeKindNameFunction GetEdgeKindName) : Name(std::move(Name)), TT(TT), PointerSize(PointerSize), - Endianness(Endianness) {} + Endianness(Endianness), GetEdgeKindName(std::move(GetEdgeKindName)) {} /// Returns the name of this graph (usually the name of the original /// underlying MemoryBuffer). @@ -810,6 +813,8 @@ /// Returns the endianness of content in this graph. support::endianness getEndianness() const { return Endianness; } + const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); } + /// Allocate a copy of the given string using the LinkGraph's allocator. /// This can be useful when renaming symbols or adding new content to the /// graph. @@ -1055,13 +1060,7 @@ } /// Dump the graph. - /// - /// If supplied, the EdgeKindToName function will be used to name edge - /// kinds in the debug output. Otherwise raw edge kind numbers will be - /// displayed. - void dump(raw_ostream &OS, - std::function EdegKindToName = - std::function()); + void dump(raw_ostream &OS); private: // Put the BumpPtrAllocator first so that we don't free any of the underlying @@ -1072,6 +1071,7 @@ Triple TT; unsigned PointerSize; support::endianness Endianness; + GetEdgeKindNameFunction GetEdgeKindName = nullptr; SectionList Sections; ExternalSymbolSet ExternalSymbols; ExternalSymbolSet AbsoluteSymbols; @@ -1387,6 +1387,10 @@ /// conservative mark-live implementation. Error markAllSymbolsLive(LinkGraph &G); +/// Create an out of range error for the given edge in the given block. +Error makeTargetOutOfRangeError(const Block &B, const Edge &E, + const char *(*getEdgeKindName)(Edge::Kind)); + /// Create a LinkGraph from the given object buffer. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -61,7 +61,7 @@ std::unique_ptr Ctx); /// Return the string name of the given MachO arm64 edge kind. -StringRef getMachOARM64RelocationKindName(Edge::Kind R); +const char *getMachOARM64RelocationKindName(Edge::Kind R); } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -18,33 +18,6 @@ namespace llvm { namespace jitlink { -namespace MachO_x86_64_Edges { - -enum MachOX86RelocationKind : Edge::Kind { - Branch32 = Edge::FirstRelocation, - Branch32ToStub, - Pointer32, - Pointer64, - Pointer64Anon, - PCRel32, - PCRel32Minus1, - PCRel32Minus2, - PCRel32Minus4, - PCRel32Anon, - PCRel32Minus1Anon, - PCRel32Minus2Anon, - PCRel32Minus4Anon, - PCRel32GOTLoad, - PCRel32GOT, - PCRel32TLV, - Delta32, - Delta64, - NegDelta32, - NegDelta64, -}; - -} // namespace MachO_x86_64_Edges - /// Create a LinkGraph from a MachO/x86-64 relocatable object. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy @@ -65,9 +38,6 @@ void link_MachO_x86_64(std::unique_ptr G, std::unique_ptr Ctx); -/// Return the string name of the given MachO x86-64 edge kind. -StringRef getMachOX86RelocationKindName(Edge::Kind R); - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -0,0 +1,328 @@ +//===-- x86_64.h - Generic JITLink x86-64 edge kinds, utilities -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing x86-64 objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { +namespace x86_64 { + +/// Represents x86-64 fixups and other x86-64-specific edge kinds. +enum EdgeKind_x86_64 : Edge::Kind { + + /// A plain 64-bit pointer value relocation. + /// + /// Fixup expression: + /// Fixup <- Target + Addend : uint64 + /// + Pointer64 = Edge::FirstRelocation, + + /// A plain 32-bit pointer value relocation. + /// + /// Fixup expression: + /// Fixup <- Target + Addend : uint32 + /// + /// Errors: + /// - The target must reside in the low 32-bits of the address space, + /// otherwise an out-of-range error will be returned. + /// + Pointer32, + + /// A 64-bit delta. + /// + /// Delta from the fixup to the target. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend : int64 + /// + Delta64, + + /// A 32-bit delta. + /// + /// Delta from the fixup to the target. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend : int64 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + Delta32, + + /// A 64-bit negative delta. + /// + /// Delta from target back to the fixup. + /// + /// Fixup expression: + /// Fixup <- Fixup - Target + Addend : int64 + /// + NegDelta64, + + /// A 32-bit negative delta. + /// + /// Delta from the target back to the fixup. + /// + /// Fixup expression: + /// Fixup <- Fixup - Target + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + NegDelta32, + + /// A 32-bit PC-relative branch. + /// + /// Represents a PC-relative call or branch to a target. This can be used to + /// identify, record, and/or patch call sites. + /// + /// The fixup expression for this kind includes an implicit offset to account + /// for the PC (unlike the Delta edges) so that a Branch32PCRel with a target + /// T and addend zero is a call/branch to the start (offset zero) of T. + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32, + + /// A 32-bit PC-relative branch to a pointer jump stub. + /// + /// The target of this relocation should be a pointer jump stub of the form: + /// + /// \code{.s} + /// .text + /// jmpq *tgtptr(%rip) + /// ; ... + /// + /// .data + /// tgtptr: + /// .quad 0 + /// \endcode + /// + /// This edge kind has the same fixup expression as BranchPCRel32, but further + /// identifies the call/branch as being to a pointer jump stub. For edges of + /// this kind the jump stub should not be bypassed (use + /// BranchPCRel32ToPtrJumpStubRelaxable for that), but the pointer location + /// target may be recorded to allow manipulation at runtime. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend - 4 : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32ToPtrJumpStub, + + /// A relaxable version of BranchPCRel32ToPtrJumpStub. + /// + /// The edge kind has the same fixup expression as BranchPCRel32ToPtrJumpStub, + /// but identifies the call/branch as being to a pointer jump stub that may be + /// bypassed if the ultimate target is within range of the fixup location. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend - 4: int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32ToPtrJumpStubRelaxable, + + /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT + /// entry for the original target. + /// + /// Indicates that this edge should be transformed into a Delta32 targeting + /// the GOT entry for the edge's current target, maintaining the same addend. + /// A GOT entry for the target should be created if one does not already + /// exist. + /// + /// Edges of this kind are usually handled by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToDelta32, + + /// A PC-relative reference to a GOT entry, relaxable if GOT entry target + /// is in-range of the fixup. + /// + /// If the GOT entry target is in-range of the fixup then the load from the + /// GOT may be replaced with a direct memory address calculation. + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + PCRel32GOTLoadRelaxable, + + /// A GOT entry getter/constructor, transformed to PCRel32ToGOTLoadRelaxable + /// pointing at the GOT entry for the original target. + /// + /// Indicates that this edge should be transformed into a + /// PC32ToGOTLoadRelaxable targeting the GOT entry for the edge's current + /// target, maintaining the same addend. A GOT entry for the target should be + /// created if one does not already exist. + /// + /// Edges of this kind are usually handled by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToPCRel32GOTLoadRelaxable, + + /// A PC-relative reference to a Thread Local Variable Pointer (TLVP) entry, + /// relaxable if the TLVP entry target is in-range of the fixup. + /// + /// If the TLVP entry target is in-range of the fixup then the load frmo the + /// TLVP may be replaced with a direct memory address calculation. + /// + /// The target of this edge must be a thread local variable entry of the form + /// .quad + /// .quad + /// .quad + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// - The target must be either external, or a TLV entry of the required + /// form, otherwise a malformed TLV entry error will be returned. + /// + PCRel32TLVPLoadRelaxable, + + /// A TLVP entry getter/constructor, transformed to + /// Delta32ToTLVPLoadRelaxable. + /// + /// Indicates that this edge should be transformed into a + /// Delta32ToTLVPLoadRelaxable targeting the TLVP entry for the edge's current + /// target. A TLVP entry for the target should be created if one does not + /// already exist. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable +}; + +/// Returns a string name for the given x86-64 edge. For debugging purposes +/// only. +const char *getEdgeKindName(Edge::Kind K); + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) { + using namespace support; + + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + + case Pointer64: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + + case Pointer32: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(B, E, getEdgeKindName); + *(ulittle32_t *)FixupPtr = Value; + break; + } + + case BranchPCRel32: + case BranchPCRel32ToPtrJumpStub: + case BranchPCRel32ToPtrJumpStubRelaxable: + case PCRel32GOTLoadRelaxable: + case PCRel32TLVPLoadRelaxable: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(B, E, getEdgeKindName); + *(little32_t *)FixupPtr = Value; + break; + } + + case Delta64: { + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + + case Delta32: { + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(B, E, getEdgeKindName); + *(little32_t *)FixupPtr = Value; + break; + } + + case NegDelta64: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + + case NegDelta32: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(B, E, getEdgeKindName); + *(little32_t *)FixupPtr = Value; + break; + } + + default: { + // If you hit this you should check that *constructor and other non-fixup + // edges have been removed prior to applying fixups. + llvm_unreachable("Graph contains edge kind with no fixup expression"); + } + } + + return Error::success(); +} + +} // namespace x86_64 +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_X86_64_H diff --git a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h --- a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -33,7 +33,7 @@ for (auto *B : Blocks) for (auto &E : B->edges()) - if (impl().isGOTEdge(E)) { + if (impl().isGOTEdgeToFix(E)) { LLVM_DEBUG({ dbgs() << " Updating GOT edge "; printEdge(dbgs(), *B, E, ""); diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -3,15 +3,23 @@ JITLinkGeneric.cpp JITLinkMemoryManager.cpp EHFrameSupport.cpp - #macho + + # Formats: + + # MachO MachO.cpp MachO_arm64.cpp MachO_x86_64.cpp MachOLinkGraphBuilder.cpp - #elf + + # ELF + ELF.cpp ELF_x86_64.cpp + # Architectures: + x86_64.cpp + ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/JITLink diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -36,7 +36,7 @@ ELF_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder(G) {} - bool isGOTEdge(Edge &E) const { + bool isGOTEdgeToFix(Edge &E) const { return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; } @@ -652,9 +652,9 @@ public: ELFLinkGraphBuilder_x86_64(StringRef FileName, const object::ELFFile &Obj) - : G(std::make_unique(FileName.str(), - Triple("x86_64-unknown-linux"), - getPointerSize(Obj), getEndianness(Obj))), + : G(std::make_unique( + FileName.str(), Triple("x86_64-unknown-linux"), getPointerSize(Obj), + getEndianness(Obj), getELFX86RelocationKindName)), Obj(Obj) {} Expected> buildGraph() { @@ -695,9 +695,6 @@ : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getELFX86RelocationKindName(R); - } static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; @@ -792,7 +789,7 @@ ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getELFX86RelocationKindName(Edge::Kind R) { +const char *getELFX86RelocationKindName(Edge::Kind R) { switch (R) { case PCRel32: return "PCRel32"; diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -251,11 +251,7 @@ return NewBlock; } -void LinkGraph::dump(raw_ostream &OS, - std::function EdgeKindToName) { - if (!EdgeKindToName) - EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; - +void LinkGraph::dump(raw_ostream &OS) { OS << "Symbols:\n"; for (auto *Sym : defined_symbols()) { OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym @@ -263,16 +259,7 @@ if (Sym->isDefined()) { for (auto &E : Sym->getBlock().edges()) { OS << " "; - StringRef EdgeName = (E.getKind() < Edge::FirstRelocation - ? getGenericEdgeKindName(E.getKind()) - : EdgeKindToName(E.getKind())); - - if (!EdgeName.empty()) - printEdge(OS, Sym->getBlock(), E, EdgeName); - else { - auto EdgeNumberString = std::to_string(E.getKind()); - printEdge(OS, Sym->getBlock(), E, EdgeNumberString); - } + printEdge(OS, Sym->getBlock(), E, getEdgeKindName(E.getKind())); OS << "\n"; } } @@ -322,6 +309,18 @@ return Error::success(); } +Error makeTargetOutOfRangeError(const Block &B, const Edge &E, + const char *(*getEdgeKindName)(Edge::Kind)) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getEdgeKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error(std::move(ErrMsg)); +} + Expected> createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { auto Magic = identify_magic(ObjectBuffer.getBuffer()); diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -76,9 +76,6 @@ // 3.1: Call OnFinalized callback, handing off allocation. void linkPhase3(std::unique_ptr Self, Error Err); - // For debug dumping of the link graph. - virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; - // Align a JITTargetAddress to conform with block alignment requirements. static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); @@ -109,8 +106,6 @@ JITLinkMemoryManager::Allocation &Alloc); void deallocateAndBailOut(Error Err); - void dumpGraph(raw_ostream &OS); - std::unique_ptr Ctx; std::unique_ptr G; PassConfiguration Passes; diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -34,14 +34,14 @@ LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); prune(*G); LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Run post-pruning passes. @@ -58,7 +58,7 @@ LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before post-allocation passes:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Run post-allocation passes. @@ -121,7 +121,7 @@ LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before pre-fixup passes:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); if (auto Err = runPasses(Passes.PreFixupPasses)) @@ -129,7 +129,7 @@ LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Fix up block content. @@ -138,7 +138,7 @@ LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); if (auto Err = runPasses(Passes.PostFixupPasses)) @@ -415,11 +415,6 @@ Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate())); } -void JITLinkerBase::dumpGraph(raw_ostream &OS) { - assert(G && "Graph is not set yet"); - G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); -} - void prune(LinkGraph &G) { std::vector Worklist; DenseSet VisitedBlocks; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -81,7 +81,8 @@ using SectionParserFunction = std::function; - MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT); + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); LinkGraph &getGraph() const { return *G; } diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -45,12 +45,13 @@ return std::move(G); } -MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, - Triple TT) +MachOLinkGraphBuilder::MachOLinkGraphBuilder( + const object::MachOObjectFile &Obj, Triple TT, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : Obj(Obj), - G(std::make_unique(std::string(Obj.getFileName()), - std::move(TT), getPointerSize(Obj), - getEndianness(Obj))) {} + G(std::make_unique( + std::string(Obj.getFileName()), std::move(TT), getPointerSize(Obj), + getEndianness(Obj), std::move(GetEdgeKindName))) {} void MachOLinkGraphBuilder::addCustomSectionParser( StringRef SectionName, SectionParserFunction Parser) { diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -26,7 +26,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin")), + : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin"), + getMachOARM64RelocationKindName), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: @@ -271,7 +272,7 @@ if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) return make_error( "Invalid relocation pair: Addend + " + - getMachOARM64RelocationKindName(*Kind)); + StringRef(getMachOARM64RelocationKindName(*Kind))); LLVM_DEBUG({ dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) @@ -410,9 +411,10 @@ MachO_arm64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder(G) {} - bool isGOTEdge(Edge &E) const { - return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || - E.getKind() == PointerToGOT; + bool isGOTEdgeToFix(Edge &E) const { + return (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || + E.getKind() == PointerToGOT) && + E.getTarget().isExternal(); } Symbol &createGOTEntry(Symbol &Target) { @@ -506,9 +508,6 @@ : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getMachOARM64RelocationKindName(R); - } static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; @@ -708,7 +707,7 @@ MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getMachOARM64RelocationKindName(Edge::Kind R) { +const char *getMachOARM64RelocationKindName(Edge::Kind R) { switch (R) { case Branch26: return "Branch26"; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "BasicGOTAndStubsBuilder.h" #include "MachOLinkGraphBuilder.h" @@ -19,69 +20,86 @@ using namespace llvm; using namespace llvm::jitlink; -using namespace llvm::jitlink::MachO_x86_64_Edges; namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin")) {} + : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin"), + x86_64::getEdgeKindName) {} private: - static Expected - getRelocationKind(const MachO::relocation_info &RI) { + enum MachONormalizedRelocationType : unsigned { + MachOBranch32, + MachOPointer32, + MachOPointer64, + MachOPointer64Anon, + MachOPCRel32, + MachOPCRel32Minus1, + MachOPCRel32Minus2, + MachOPCRel32Minus4, + MachOPCRel32Anon, + MachOPCRel32Minus1Anon, + MachOPCRel32Minus2Anon, + MachOPCRel32Minus4Anon, + MachOPCRel32GOTLoad, + MachOPCRel32GOT, + MachOPCRel32TLV, + MachOSubtractor32, + MachOSubtractor64, + }; + + static Expected + getRelocKind(const MachO::relocation_info &RI) { switch (RI.r_type) { case MachO::X86_64_RELOC_UNSIGNED: if (!RI.r_pcrel) { if (RI.r_length == 3) - return RI.r_extern ? Pointer64 : Pointer64Anon; + return RI.r_extern ? MachOPointer64 : MachOPointer64Anon; else if (RI.r_extern && RI.r_length == 2) - return Pointer32; + return MachOPointer32; } break; case MachO::X86_64_RELOC_SIGNED: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32 : PCRel32Anon; + return RI.r_extern ? MachOPCRel32 : MachOPCRel32Anon; break; case MachO::X86_64_RELOC_BRANCH: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Branch32; + return MachOBranch32; break; case MachO::X86_64_RELOC_GOT_LOAD: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32GOTLoad; + return MachOPCRel32GOTLoad; break; case MachO::X86_64_RELOC_GOT: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32GOT; + return MachOPCRel32GOT; break; case MachO::X86_64_RELOC_SUBTRACTOR: - // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. - // Initially represent SUBTRACTOR relocations with 'Delta'. They may - // be turned into NegDelta by parsePairRelocation. if (!RI.r_pcrel && RI.r_extern) { if (RI.r_length == 2) - return Delta32; + return MachOSubtractor32; else if (RI.r_length == 3) - return Delta64; + return MachOSubtractor64; } break; case MachO::X86_64_RELOC_SIGNED_1: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + return RI.r_extern ? MachOPCRel32Minus1 : MachOPCRel32Minus1Anon; break; case MachO::X86_64_RELOC_SIGNED_2: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + return RI.r_extern ? MachOPCRel32Minus2 : MachOPCRel32Minus2Anon; break; case MachO::X86_64_RELOC_SIGNED_4: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + return RI.r_extern ? MachOPCRel32Minus4 : MachOPCRel32Minus4Anon; break; case MachO::X86_64_RELOC_TLV: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32TLV; + return MachOPCRel32TLV; break; } @@ -95,20 +113,19 @@ ", length=" + formatv("{0:d}", RI.r_length)); } - using PairRelocInfo = std::tuple; + using PairRelocInfo = std::tuple; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. - Expected - parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, - const MachO::relocation_info &SubRI, - JITTargetAddress FixupAddress, const char *FixupContent, - object::relocation_iterator &UnsignedRelItr, - object::relocation_iterator &RelEnd) { + Expected parsePairRelocation( + Block &BlockToFix, MachONormalizedRelocationType SubtractorKind, + const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, + const char *FixupContent, object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { using namespace support; - assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || - (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + assert(((SubtractorKind == MachOSubtractor32 && SubRI.r_length == 2) || + (SubtractorKind == MachOSubtractor64 && SubRI.r_length == 3)) && "Subtractor kind should match length"); assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); @@ -158,17 +175,18 @@ FixupValue -= ToSymbol->getAddress(); } - MachOX86RelocationKind DeltaKind; + Edge::Kind DeltaKind; Symbol *TargetSymbol; uint64_t Addend; if (&BlockToFix == &FromSymbol->getAddressable()) { TargetSymbol = ToSymbol; - DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + DeltaKind = (SubRI.r_length == 3) ? x86_64::Delta64 : x86_64::Delta32; Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. } else if (&BlockToFix == &ToSymbol->getAddressable()) { TargetSymbol = FromSymbol; - DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + DeltaKind = + (SubRI.r_length == 3) ? x86_64::NegDelta64 : x86_64::NegDelta32; Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { // BlockToFix was neither FromSymbol nor ToSymbol. @@ -218,11 +236,6 @@ MachO::relocation_info RI = getRelocationInfo(RelItr); - // Sanity check the relocation kind. - auto Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); - // Find the address of the value to fix up. JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; @@ -251,111 +264,149 @@ const char *FixupContent = BlockToFix->getContent().data() + (FixupAddress - BlockToFix->getAddress()); + size_t FixupOffset = FixupAddress - BlockToFix->getAddress(); + // The target symbol and addend will be populated by the switch below. Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; - switch (*Kind) { - case Branch32: - case PCRel32: - case PCRel32GOTLoad: - case PCRel32GOT: + // Sanity check the relocation kind. + auto MachORelocKind = getRelocKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); + + Edge::Kind Kind = Edge::Invalid; + + switch (*MachORelocKind) { + case MachOBranch32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent; + Kind = x86_64::BranchPCRel32; + break; + case MachOPCRel32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent - 4; + Kind = x86_64::Delta32; + break; + case MachOPCRel32GOTLoad: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const little32_t *)FixupContent; + Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; + if (FixupOffset < 3) + return make_error("GOTLD at invalid offset " + + formatv("{0}", FixupOffset)); break; - case Pointer32: + case MachOPCRel32GOT: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent - 4; + Kind = x86_64::RequestGOTAndTransformToDelta32; + break; + case MachOPointer32: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; + Kind = x86_64::Pointer32; break; - case Pointer64: + case MachOPointer64: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; + Kind = x86_64::Pointer64; break; - case Pointer64Anon: { + case MachOPointer64Anon: { JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); Addend = TargetAddress - TargetSymbol->getAddress(); + Kind = x86_64::Pointer64; break; } - case PCRel32Minus1: - case PCRel32Minus2: - case PCRel32Minus4: + case MachOPCRel32Minus1: + case MachOPCRel32Minus2: + case MachOPCRel32Minus4: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); - Addend = *(const little32_t *)FixupContent + - (1 << (*Kind - PCRel32Minus1)); + Addend = *(const little32_t *)FixupContent - 4; + // - + // (1 << (*MachORelocKind - MachOPCRel32Minus1)); + Kind = x86_64::Delta32; break; - case PCRel32Anon: { + case MachOPCRel32Anon: { JITTargetAddress TargetAddress = FixupAddress + 4 + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); - Addend = TargetAddress - TargetSymbol->getAddress(); + Addend = TargetAddress - TargetSymbol->getAddress() - 4; + Kind = x86_64::Delta32; break; } - case PCRel32Minus1Anon: - case PCRel32Minus2Anon: - case PCRel32Minus4Anon: { + case MachOPCRel32Minus1Anon: + case MachOPCRel32Minus2Anon: + case MachOPCRel32Minus4Anon: { JITTargetAddress Delta = - static_cast(1ULL << (*Kind - PCRel32Minus1Anon)); + 4 + static_cast( + 1ULL << (*MachORelocKind - MachOPCRel32Minus1Anon)); JITTargetAddress TargetAddress = - FixupAddress + 4 + Delta + *(const little32_t *)FixupContent; + FixupAddress + Delta + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); - Addend = TargetAddress - TargetSymbol->getAddress(); + Addend = TargetAddress - TargetSymbol->getAddress() - Delta; + Kind = x86_64::Delta32; break; } - case Delta32: - case Delta64: { + case MachOSubtractor32: + case MachOSubtractor64: { // We use Delta32/Delta64 to represent SUBTRACTOR relocations. // parsePairRelocation handles the paired reloc, and returns the // edge kind to be used (either Delta32/Delta64, or // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, - FixupContent, ++RelItr, RelEnd); + parsePairRelocation(*BlockToFix, *MachORelocKind, RI, + FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + std::tie(Kind, TargetSymbol, Addend) = *PairInfo; assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } - case PCRel32TLV: + case MachOPCRel32TLV: return make_error( "MachO TLV relocations not yet supported"); - default: - llvm_unreachable("Special relocation kind should not appear in " - "mach-o file"); } LLVM_DEBUG({ dbgs() << " "; - Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *BlockToFix, GE, - getMachOX86RelocationKindName(*Kind)); + printEdge(dbgs(), *BlockToFix, GE, x86_64::getEdgeKindName(Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); } } @@ -372,33 +423,37 @@ MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder(G) {} - bool isGOTEdge(Edge &E) const { - return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + bool isGOTEdgeToFix(Edge &E) const { + return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 || + E.getKind() == + x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; } Symbol &createGOTEntry(Symbol &Target) { auto &GOTEntryBlock = G.createContentBlock( getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); - GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0); return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && - "Not a GOT edge?"); - // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is - // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to - // check for GOT optimization opportunities in the - // optimizeMachO_x86_64_GOTAndStubs pass below. - if (E.getKind() == PCRel32GOT) - E.setKind(PCRel32); - + // Fix the edge kind. + switch (E.getKind()) { + case x86_64::RequestGOTAndTransformToDelta32: + E.setKind(x86_64::Delta32); + break; + case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable: + E.setKind(x86_64::PCRel32GOTLoadRelaxable); + break; + default: + llvm_unreachable("Not a GOT transform edge"); + } + // Fix the target, leave the addend as-is. E.setTarget(GOTEntry); - // Leave the edge addend as-is. } bool isExternalBranchEdge(Edge &E) { - return E.getKind() == Branch32 && !E.getTarget().isDefined(); + return E.getKind() == x86_64::BranchPCRel32 && E.getTarget().isExternal(); } Symbol &createStub(Symbol &Target) { @@ -406,18 +461,19 @@ G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. auto &GOTEntrySymbol = getGOTEntrySymbol(Target); - StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); + StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4); return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } void fixExternalBranchEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); - assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && + "BranchPCRel32 edge has unexpected addend value"); - // Set the edge kind to Branch32ToStub. We will use this to check for stub - // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass - // below. - E.setKind(Branch32ToStub); + // Set the edge kind to BranchPCRel32ToPtrJumpStubRelaxable. We will use + // this to check for stub optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + E.setKind(x86_64::BranchPCRel32ToPtrJumpStubRelaxable); E.setTarget(Stub); } @@ -462,13 +518,9 @@ for (auto *B : G.blocks()) for (auto &E : B->edges()) - if (E.getKind() == PCRel32GOTLoad) { + if (E.getKind() == x86_64::PCRel32GOTLoadRelaxable) { assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); - // Switch the edge kind to PCRel32: Whether we change the edge target - // or not this will be the desired kind. - E.setKind(PCRel32); - // Optimize GOT references. auto &GOTBlock = E.getTarget().getBlock(); assert(GOTBlock.getSize() == G.getPointerSize() && @@ -491,22 +543,18 @@ if (Displacement >= std::numeric_limits::min() && Displacement <= std::numeric_limits::max()) { E.setTarget(GOTTarget); + E.setKind(x86_64::Delta32); + E.setAddend(E.getAddend() - 4); auto *BlockData = reinterpret_cast( const_cast(B->getContent().data())); BlockData[E.getOffset() - 2] = 0x8d; LLVM_DEBUG({ dbgs() << " Replaced GOT load wih LEA:\n "; - printEdge(dbgs(), *B, E, - getMachOX86RelocationKindName(E.getKind())); + printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); dbgs() << "\n"; }); } - } else if (E.getKind() == Branch32ToStub) { - - // Switch the edge kind to PCRel32: Whether we change the edge target - // or not this will be the desired kind. - E.setKind(Branch32); - + } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) { auto &StubBlock = E.getTarget().getBlock(); assert(StubBlock.getSize() == sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) && @@ -527,11 +575,11 @@ int64_t Displacement = TargetAddr - EdgeAddr + 4; if (Displacement >= std::numeric_limits::min() && Displacement <= std::numeric_limits::max()) { + E.setKind(x86_64::BranchPCRel32); E.setTarget(GOTTarget); LLVM_DEBUG({ dbgs() << " Replaced stub branch with direct branch:\n "; - printEdge(dbgs(), *B, E, - getMachOX86RelocationKindName(E.getKind())); + printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); dbgs() << "\n"; }); } @@ -553,104 +601,10 @@ : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getMachOX86RelocationKindName(R); - } - - static Error targetOutOfRangeError(const Block &B, const Edge &E) { - std::string ErrMsg; - { - raw_string_ostream ErrStream(ErrMsg); - ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); - ErrStream << "\n"; - } - return make_error(std::move(ErrMsg)); - } Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { - - using namespace support; - - char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch32: - case PCRel32: - case PCRel32Anon: { - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case PCRel32Minus1: - case PCRel32Minus2: - case PCRel32Minus4: { - int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case PCRel32Minus1Anon: - case PCRel32Minus2Anon: - case PCRel32Minus4Anon: { - int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return targetOutOfRangeError(B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - default: - llvm_unreachable("Unrecognized edge kind"); - } - - return Error::success(); + return x86_64::applyFixup(B, E, BlockWorkingMem); } - - uint64_t NullValue = 0; }; Expected> @@ -669,8 +623,9 @@ if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { // Add eh-frame passses. Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); - Config.PrePrunePasses.push_back(EHFrameEdgeFixer( - "__eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + Config.PrePrunePasses.push_back( + EHFrameEdgeFixer("__eh_frame", G->getPointerSize(), x86_64::Delta64, + x86_64::Delta32, x86_64::NegDelta32)); // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) @@ -695,52 +650,5 @@ MachOJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getMachOX86RelocationKindName(Edge::Kind R) { - switch (R) { - case Branch32: - return "Branch32"; - case Branch32ToStub: - return "Branch32ToStub"; - case Pointer32: - return "Pointer32"; - case Pointer64: - return "Pointer64"; - case Pointer64Anon: - return "Pointer64Anon"; - case PCRel32: - return "PCRel32"; - case PCRel32Minus1: - return "PCRel32Minus1"; - case PCRel32Minus2: - return "PCRel32Minus2"; - case PCRel32Minus4: - return "PCRel32Minus4"; - case PCRel32Anon: - return "PCRel32Anon"; - case PCRel32Minus1Anon: - return "PCRel32Minus1Anon"; - case PCRel32Minus2Anon: - return "PCRel32Minus2Anon"; - case PCRel32Minus4Anon: - return "PCRel32Minus4Anon"; - case PCRel32GOTLoad: - return "PCRel32GOTLoad"; - case PCRel32GOT: - return "PCRel32GOT"; - case PCRel32TLV: - return "PCRel32TLV"; - case Delta32: - return "Delta32"; - case Delta64: - return "Delta64"; - case NegDelta32: - return "NegDelta32"; - case NegDelta64: - return "NegDelta64"; - default: - return getGenericEdgeKindName(static_cast(R)); - } -} - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -0,0 +1,58 @@ +//===----- x86_64.cpp - Generic JITLink x86-64 edge kinds, utilities ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing x86-64 objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { +namespace x86_64 { + +const char *getEdgeKindName(Edge::Kind K) { + switch (K) { + case Pointer64: + return "Pointer64"; + case Pointer32: + return "Pointer32"; + case Delta64: + return "Delta64"; + case Delta32: + return "Delta32"; + case NegDelta64: + return "NegDelta64"; + case NegDelta32: + return "NegDelta32"; + case BranchPCRel32: + return "BranchPCRel32"; + case BranchPCRel32ToPtrJumpStub: + return "BranchPCRel32ToPtrJumpStub"; + case BranchPCRel32ToPtrJumpStubRelaxable: + return "BranchPCRel32ToPtrJumpStubRelaxable"; + case RequestGOTAndTransformToDelta32: + return "RequestGOTAndTransformToDelta32"; + case PCRel32GOTLoadRelaxable: + return "PCRel32GOTLoadRelaxable"; + case RequestGOTAndTransformToPCRel32GOTLoadRelaxable: + return "RequestGOTAndTransformToPCRel32GOTLoadRelaxable"; + case PCRel32TLVPLoadRelaxable: + return "PCRel32TLVPLoadRelaxable"; + case RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable: + return "RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable"; + default: + return getGenericEdgeKindName(static_cast(K)); + } +} + +} // end namespace x86_64 +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp --- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp +++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp @@ -25,7 +25,8 @@ TEST(LinkGraphTest, Construction) { // Check that LinkGraph construction works as expected. - LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little); + LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little, + getGenericEdgeKindName); EXPECT_EQ(G.getName(), "foo"); EXPECT_EQ(G.getTargetTriple().str(), "x86_64-apple-darwin"); EXPECT_EQ(G.getPointerSize(), 8U); @@ -38,7 +39,8 @@ TEST(LinkGraphTest, AddressAccess) { // Check that we can get addresses for blocks, symbols, and edges. - LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little); + LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little, + getGenericEdgeKindName); auto Sec1 = G.createSection("__data.1", RWFlags); auto &B1 = G.createContentBlock(Sec1, BlockContent, 0x1000, 8, 0); @@ -54,7 +56,8 @@ TEST(LinkGraphTest, BlockAndSymbolIteration) { // Check that we can iterate over blocks within Sections and across sections. - LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little); + LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little, + getGenericEdgeKindName); auto &Sec1 = G.createSection("__data.1", RWFlags); auto &B1 = G.createContentBlock(Sec1, BlockContent, 0x1000, 8, 0); auto &B2 = G.createContentBlock(Sec1, BlockContent, 0x2000, 8, 0); @@ -106,7 +109,8 @@ 0x1C, 0x1D, 0x1E, 0x1F, 0x00}; StringRef BlockContent(BlockContentBytes); - LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little); + LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little, + getGenericEdgeKindName); auto &Sec = G.createSection("__data", RWFlags); // Create the block to split.