diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -1709,6 +1709,30 @@ Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, const Edge &E); +static inline void visitEdge(LinkGraph &G, Block *B, Edge &E) {} + +template +static void visitEdge(LinkGraph &G, Block *B, Edge &E, FixerT &&Fixer, + FixerTs &&...Fixers) { + if (!Fixer.visitEdge(G, B, E)) + visitEdge(G, B, E, std::forward(Fixers)...); +} + +/// Visits edges exist in graph by Fixers. +/// +/// Note: that if a fixer fixes the edge successfully, +/// the rest of the fixers will not visit this edge. +template +void visitExistingEdges(LinkGraph &G, FixerTs &&...Fixers) { + // We're going to be adding new blocks, but we don't want to iterate over + // the new ones, so build a worklist. + std::vector Worklist(G.blocks().begin(), G.blocks().end()); + + for (auto *B : Worklist) + for (auto &E : B->edges()) + visitEdge(G, B, E, std::forward(Fixers)...); +} + /// Create a LinkGraph from the given object buffer. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -21,7 +21,7 @@ #include "ELFLinkGraphBuilder.h" #include "JITLinkGeneric.h" #include "PerGraphGOTAndPLTStubsBuilder.h" -#include "PerGraphTLSInfoEntryBuilder.h" +#include "TableManager.h" #define DEBUG_TYPE "jitlink" @@ -35,172 +35,186 @@ constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO"; -class PerGraphTLSInfoBuilder_ELF_x86_64 - : public PerGraphTLSInfoEntryBuilder { -public: - static const uint8_t TLSInfoEntryContent[16]; - using PerGraphTLSInfoEntryBuilder< - PerGraphTLSInfoBuilder_ELF_x86_64>::PerGraphTLSInfoEntryBuilder; - - bool isTLSEdgeToFix(Edge &E) { - return E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32; - } - - Symbol &createTLSInfoEntry(Symbol &Target) { - // the TLS Info entry's key value will be written by the fixTLVSectionByName - // pass, so create mutable content. - auto &TLSInfoEntry = G.createMutableContentBlock( - getTLSInfoSection(), G.allocateContent(getTLSInfoEntryContent()), 0, 8, - 0); - TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0); - return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); - } - - void fixTLSEdge(Edge &E, Symbol &Target) { - if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { - E.setTarget(Target); - E.setKind(x86_64::Delta32); - } - } - - Section &getTLSInfoSection() const { - if (!TLSInfoSection) - TLSInfoSection = &G.createSection(ELFTLSInfoSectionName, MemProt::Read); - return *TLSInfoSection; - } - -private: - ArrayRef getTLSInfoEntryContent() { - return {reinterpret_cast(TLSInfoEntryContent), - sizeof(TLSInfoEntryContent)}; - } - - mutable Section *TLSInfoSection = nullptr; -}; - -const uint8_t PerGraphTLSInfoBuilder_ELF_x86_64::TLSInfoEntryContent[16] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ -}; - -class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> { +class GOTTableManager_ELF_x86_64 + : public TableManager { public: static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[6]; - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; + // Nice name for table + StringRef getTableName() { return "GOT"; } - bool isGOTEdgeToFix(Edge &E) const { - if (E.getKind() == x86_64::Delta64FromGOT) { - // We need to make sure that the GOT section exists, but don't otherwise - // need to fix up this edge. - getGOTSection(); + bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = E.getKind(); + switch (E.getKind()) { + case x86_64::Delta64FromGOT: { + // we need to make sure that the GOT section exists, but don't otherwise + // need to fix up this edge + getGOTSection(G); return false; } - return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 || - E.getKind() == x86_64::RequestGOTAndTransformToDelta64 || - E.getKind() == - x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable || - E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT || - E.getKind() == - x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); - GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary - // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now: - // We will use the kind to check for GOT optimization opportunities in the - // optimizeMachO_x86_64_GOTAndStubs pass below. - // If it's a GOT64 leave it as is. - switch (E.getKind()) { case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable: - E.setKind(x86_64::PCRel32GOTLoadREXRelaxable); + KindToSet = x86_64::PCRel32GOTLoadREXRelaxable; break; case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable: - E.setKind(x86_64::PCRel32GOTLoadRelaxable); + KindToSet = x86_64::PCRel32GOTLoadRelaxable; break; case x86_64::RequestGOTAndTransformToDelta64: - E.setKind(x86_64::Delta64); + KindToSet = x86_64::Delta64; break; case x86_64::RequestGOTAndTransformToDelta64FromGOT: - E.setKind(x86_64::Delta64FromGOT); + KindToSet = x86_64::Delta64FromGOT; break; case x86_64::RequestGOTAndTransformToDelta32: - E.setKind(x86_64::Delta32); + KindToSet = x86_64::Delta32; break; default: - llvm_unreachable("Unexpected GOT edge kind"); + return false; } - - E.setTarget(GOTEntry); - // Leave the edge addend as-is. + LLVM_DEBUG({ + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << formatv("{0:x}", B->getFixupAddress(E)) << " (" + << formatv("{0:x}", B->getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + return true; } - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined(); + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(G), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = - G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4); - return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = &G.createSection(ELFGOTSectionName, MemProt::Read); + return *GOTSection; + } + ArrayRef getGOTEntryBlockContent() const { + return {reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; } + Section *GOTSection = nullptr; +}; +const uint8_t GOTTableManager_ELF_x86_64::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +class PLTTableManager_ELF_x86_64 + : public TableManager { +public: + PLTTableManager_ELF_x86_64(GOTTableManager_ELF_x86_64 &GOTTable) + : GOTTable(GOTTable) {} - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?"); + StringRef getTableName() { return "PLT"; } - // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to be - // optimized when the target is in-range. - E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable); - E.setTarget(Stub); + static const uint8_t StubContent[6]; + bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) { + LLVM_DEBUG({ + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << formatv("{0:x}", B->getFixupAddress(E)) << " (" + << formatv("{0:x}", B->getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to + // be optimized when the target is in-range. + E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable); + return true; + } + return false; } -private: - Section &getGOTSection() const { - if (!GOTSection) - GOTSection = &G.createSection(ELFGOTSectionName, MemProt::Read); - return *GOTSection; + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &StubContentBlock = G.createContentBlock( + getStubsSection(G), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = GOTTable.getEntryForTarget(G, Target); + StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } - Section &getStubsSection() const { +private: + Section &getStubsSection(LinkGraph &G) { if (!StubsSection) StubsSection = &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); return *StubsSection; } - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } - ArrayRef getStubBlockContent() { return {reinterpret_cast(StubContent), sizeof(StubContent)}; } - mutable Section *GOTSection = nullptr; - mutable Section *StubsSection = nullptr; + Section *StubsSection = nullptr; + GOTTableManager_ELF_x86_64 &GOTTable; }; +const uint8_t PLTTableManager_ELF_x86_64::StubContent[6] = {0xFF, 0x25, 0x00, + 0x00, 0x00, 0x00}; -} // namespace +class TLSInfoTableManager_ELF_x86_64 + : public TableManager { +public: + static const uint8_t TLSInfoEntryContent[16]; -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] = - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { - 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; + StringRef getTableName() { return "TLSInfo"; } + + bool fixEdgeKind(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { + LLVM_DEBUG({ + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << formatv("{0:x}", B->getFixupAddress(E)) << " (" + << formatv("{0:x}", B->getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(x86_64::Delta32); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + // the TLS Info entry's key value will be written by the fixTLVSectionByName + // pass, so create mutable content. + auto &TLSInfoEntry = G.createMutableContentBlock( + getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 0, 8, + 0); + TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0); + return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); + } + +private: + Section &getTLSInfoSection(LinkGraph &G) { + if (!TLSInfoTable) + TLSInfoTable = &G.createSection(ELFTLSInfoSectionName, MemProt::Read); + return *TLSInfoTable; + } + + ArrayRef getTLSInfoEntryContent() const { + return {reinterpret_cast(TLSInfoEntryContent), + sizeof(TLSInfoEntryContent)}; + } + + Section *TLSInfoTable = nullptr; +}; + +const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ +}; + +Error buildTables_ELF_x86_64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + GOTTableManager_ELF_x86_64 GOT; + PLTTableManager_ELF_x86_64 PLT(GOT); + TLSInfoTableManager_ELF_x86_64 TLSInfo; + visitExistingEdges(G, GOT, PLT, TLSInfo); + return Error::success(); +} +} // namespace static const char *getELFX86_64RelocName(uint32_t Type) { switch (Type) { @@ -494,11 +508,8 @@ else Config.PrePrunePasses.push_back(markAllSymbolsLive); - // Add an in-place GOT/Stubs pass. - - Config.PostPrunePasses.push_back(PerGraphTLSInfoBuilder_ELF_x86_64::asPass); - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass); + // Add an in-place GOT/Stubs/TLSInfoEntry build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_x86_64); // Resolve any external section start / end symbols. Config.PostAllocationPasses.push_back( diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h deleted file mode 100644 --- a/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h +++ /dev/null @@ -1,78 +0,0 @@ -//===---------------- PerGraphTLSInfoEntryBuilder.h -------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Construct Thread local storage info entry for each graph. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHTLSINFOENTRYBUILDER_H -#define LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHTLSINFOENTRYBUILDER_H - -#include "llvm/ExecutionEngine/JITLink/JITLink.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "jitlink" -namespace llvm { -namespace jitlink { - -template class PerGraphTLSInfoEntryBuilder { -public: - PerGraphTLSInfoEntryBuilder(LinkGraph &G) : G(G) {} - static Error asPass(LinkGraph &G) { return BuilderImplT(G).run(); } - - Error run() { - LLVM_DEBUG(dbgs() << "Running Per-Graph TLS Info entry builder:\n "); - - std::vector Worklist(G.blocks().begin(), G.blocks().end()); - - for (auto *B : Worklist) - for (auto &E : B->edges()) { - if (impl().isTLSEdgeToFix(E)) { - LLVM_DEBUG({ - dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) - << " edge at " << formatv("{0:x}", B->getFixupAddress(E)) - << " (" << formatv("{0:x}", B->getAddress()) << " + " - << formatv("{0:x}", E.getOffset()) << ")\n"; - }); - impl().fixTLSEdge(E, getTLSInfoEntry(E.getTarget())); - } - } - return Error::success(); - } - -protected: - LinkGraph &G; - - Symbol &getTLSInfoEntry(Symbol &Target) { - assert(Target.hasName() && "TLS edge cannot point to anonymous target"); - auto TLSInfoEntryI = TLSInfoEntries.find(Target.getName()); - if (TLSInfoEntryI == TLSInfoEntries.end()) { - auto &TLSInfoEntry = impl().createTLSInfoEntry(Target); - LLVM_DEBUG({ - dbgs() << " Created TLS Info entry for " << Target.getName() << ": " - << TLSInfoEntry << "\n"; - }); - TLSInfoEntryI = - TLSInfoEntries.insert(std::make_pair(Target.getName(), &TLSInfoEntry)) - .first; - } - assert(TLSInfoEntryI != TLSInfoEntries.end() && - "Could not get TLSInfo symbol"); - LLVM_DEBUG({ - dbgs() << " Using TLS Info entry" << *TLSInfoEntryI->second << "\n"; - }); - return *TLSInfoEntryI->second; - } - -private: - DenseMap TLSInfoEntries; - BuilderImplT &impl() { return static_cast(*this); } -}; -} // namespace jitlink -} // namespace llvm -#endif diff --git a/llvm/lib/ExecutionEngine/JITLink/TableManager.h b/llvm/lib/ExecutionEngine/JITLink/TableManager.h new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/TableManager.h @@ -0,0 +1,75 @@ +//===---------------------- TableManager.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Fix edge for edge that needs an entry to reference the target symbol +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_TABLEMANAGER_H +#define LLVM_EXECUTIONENGINE_JITLINK_TABLEMANAGER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +/// Table like section manager +template class TableManager { +public: + /// Visit edge, return true if the edge was dealt with, otherwise return + /// false(let other managers to visit). + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (impl().fixEdgeKind(G, B, E)) { + fixTarget(G, E); + return true; + } + return false; + } + + /// Return the constructed entry + /// + /// Use parameter G to construct the entry for target symbol + Symbol &getEntryForTarget(LinkGraph &G, Symbol &Target) { + assert(Target.hasName() && "Edge cannot point to anonymous target"); + + auto EntryI = Entries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (EntryI == Entries.end()) { + auto &Entry = impl().createEntry(G, Target); + LLVM_DEBUG({ + dbgs() << " Created" << impl().getTableName() << "entry for " + << Target.getName() << ": " << Entry << "\n"; + }); + EntryI = Entries.insert(std::make_pair(Target.getName(), &Entry)).first; + } + + assert(EntryI != Entries.end() && "Could not get entry symbol"); + LLVM_DEBUG({ + dbgs() << " Using " << impl().getTableName() << " entry " + << *EntryI->second << "\n"; + }); + return *EntryI->second; + } + +private: + void fixTarget(LinkGraph &G, Edge &E) { + E.setTarget(getEntryForTarget(G, E.getTarget())); + } + + TableManagerImplT &impl() { return static_cast(*this); } + DenseMap Entries; +}; + +} // namespace jitlink +} // namespace llvm + +#endif