diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -13,7 +13,9 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H #define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H +#include "TableManager.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" namespace llvm { namespace jitlink { @@ -49,6 +51,125 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E); +/// AArch64 null pointer content. +extern const uint8_t NullGOTEntryContent[8]; + +/// AArch64 PLT stub content. +extern const uint8_t StubContent[8]; + +/// Global Offset Table Builder. +class GOTTableManager : public TableManager { +public: + static StringRef getSectionName() { return "$__GOT"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = Edge::Invalid; + const char *BlockWorkingMem = B->getContent().data(); + const char *FixupPtr = BlockWorkingMem + E.getOffset(); + + switch (E.getKind()) { + case aarch64::GOTPage21: + case aarch64::TLVPage21: { + KindToSet = aarch64::Page21; + break; + } + case aarch64::GOTPageOffset12: + case aarch64::TLVPageOffset12: { + KindToSet = aarch64::PageOffset12; + uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr; + assert(E.getAddend() == 0 && + "GOTPageOffset12/TLVPageOffset12 with non-zero addend"); + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + break; + } + case aarch64::PointerToGOT: { + KindToSet = aarch64::Delta64; + break; + } + default: + return false; + } + assert(KindToSet != Edge::Invalid && + "Fell through switch, but no new kind to set"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(G), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *GOTSection; + } + + ArrayRef getGOTEntryBlockContent() { + return {reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; + } + + Section *GOTSection = nullptr; +}; + +/// Procedure Linkage Table Builder. +class PLTTableManager : public TableManager { +public: + PLTTableManager(GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined()) { + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &StubContentBlock = G.createContentBlock( + getStubsSection(G), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = GOT.getEntryForTarget(G, Target); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + +public: + Section &getStubsSection(LinkGraph &G) { + if (!StubsSection) + StubsSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *StubsSection; + } + + ArrayRef getStubBlockContent() { + return {reinterpret_cast(StubContent), sizeof(StubContent)}; + } + + GOTTableManager &GOT; + Section *StubsSection = nullptr; +}; + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -21,15 +21,12 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/MathExtras.h" -#include "PerGraphGOTAndPLTStubsBuilder.h" - #define DEBUG_TYPE "jitlink" using namespace llvm; using namespace llvm::jitlink; -namespace llvm { -namespace jitlink { +namespace { class ELFJITLinker_aarch64 : public JITLinker { friend class JITLinker; @@ -293,90 +290,19 @@ aarch64::getEdgeKindName) {} }; -class PerGraphGOTAndPLTStubsBuilder_ELF_arm64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_arm64> { -public: - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_arm64>::PerGraphGOTAndPLTStubsBuilder; - - bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == aarch64::GOTPage21) { - E.setKind(aarch64::Page21); - E.setTarget(GOTEntry); - } else if (E.getKind() == aarch64::GOTPageOffset12) { - E.setKind(aarch64::PageOffset12); - E.setTarget(GOTEntry); - } else - llvm_unreachable("Not a GOT edge?"); - } - - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); - } - - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = G.createContentBlock( - getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); - } - - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == aarch64::Branch26 && "Not a Branch26 edge?"); - assert(E.getAddend() == 0 && "Branch26 edge has non-zero addend?"); - E.setTarget(Stub); - } - -private: - Section &getGOTSection() { - if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); - return *GOTSection; - } - - Section &getStubsSection() { - if (!StubsSection) - StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); - return *StubsSection; - } - - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } +Error buildTables_ELF_aarch64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); - ArrayRef getStubBlockContent() { - return {reinterpret_cast(StubContent), sizeof(StubContent)}; - } + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[8]; - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; +} // namespace -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::NullGOTEntryContent[8] = - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, - 0x00, 0x02, 0x1f, 0xd6 // BR x16 -}; +namespace llvm { +namespace jitlink { Expected> createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { @@ -404,18 +330,21 @@ PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); + + // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); - } - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_ELF_arm64::asPass); + // Add an in-place GOT/Stubs build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); + } if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -15,7 +15,6 @@ #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "MachOLinkGraphBuilder.h" -#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" @@ -506,103 +505,20 @@ unsigned NumSymbols = 0; }; -class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64> { -public: - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder; - - bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12 || - E.getKind() == aarch64::TLVPage21 || - E.getKind() == aarch64::TLVPageOffset12 || - E.getKind() == aarch64::PointerToGOT; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12 || - E.getKind() == aarch64::TLVPage21 || - E.getKind() == aarch64::TLVPageOffset12) { - // Update the target, but leave the edge addend as-is. - E.setTarget(GOTEntry); - } else if (E.getKind() == aarch64::PointerToGOT) { - E.setTarget(GOTEntry); - E.setKind(aarch64::Delta32); - } else - llvm_unreachable("Not a GOT edge?"); - } - - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); - } - - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = G.createContentBlock( - getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); - } - - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == aarch64::Branch26 && "Not a Branch32 edge?"); - assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); - E.setTarget(Stub); - } - -private: - Section &getGOTSection() { - if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); - return *GOTSection; - } - - Section &getStubsSection() { - if (!StubsSection) - StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); - return *StubsSection; - } - - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } - - ArrayRef getStubBlockContent() { - return {reinterpret_cast(StubContent), sizeof(StubContent)}; - } - - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[8]; - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; - -const uint8_t - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, - 0x00, 0x02, 0x1f, 0xd6 // BR x16 -}; - } // namespace namespace llvm { namespace jitlink { +Error buildTables_MachO_arm64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} + class MachOJITLinker_arm64 : public JITLinker { friend class JITLinker; @@ -654,8 +570,7 @@ aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass); + Config.PostPrunePasses.push_back(buildTables_MachO_arm64); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -80,9 +80,7 @@ *(ulittle64_t *)FixupPtr = Value; break; } - case Page21: - case TLVPage21: - case GOTPage21: { + case Page21: { assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && "GOTPAGE21 with non-zero addend"); uint64_t TargetPage = @@ -119,21 +117,6 @@ *(ulittle32_t *)FixupPtr = FixedInstr; break; } - case TLVPageOffset12: - case GOTPageOffset12: { - assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xfffffc00) == 0xf9400000 && - "RawInstr isn't a 64-bit LDR immediate"); - - uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; - assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); - uint32_t EncodedImm = (TargetOffset >> 3) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } case LDRLiteral19: { assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); @@ -170,6 +153,15 @@ *(little64_t *)FixupPtr = Value; break; } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } default: return make_error( "In graph " + G.getName() + ", section " + B.getSection().getName() + @@ -179,6 +171,14 @@ return Error::success(); } +const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const uint8_t StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; + const char *getEdgeKindName(Edge::Kind R) { switch (R) { case Branch26: