diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -299,6 +299,14 @@ /// only. const char *getEdgeKindName(Edge::Kind K); +/// Optimize the GOT and Stub relocations if the edge target address is in range +/// 1. PCRel32GOTLoadRelaxable. For this edge kind, if the target is in range, +/// then replace GOT load with lea +/// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is +/// in range, replace a indirect jump by plt stub with a direct jump to the +/// target +Error optimize_x86_64_GOTAndStubs(LinkGraph &G); + /// Returns true if the given uint64_t value is in range for a uint32_t. inline bool isInRangeForImmU32(uint64_t Value) { return Value <= std::numeric_limits::max(); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -150,82 +150,6 @@ const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; -static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { - LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); - - for (auto *B : G.blocks()) - for (auto &E : B->edges()) - if (E.getKind() == x86_64::PCRel32GOTLoadRelaxable) { - // Replace GOT load with LEA only for MOVQ instructions. - constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; - if (E.getOffset() < 3 || - strncmp(B->getContent().data() + E.getOffset() - 3, - reinterpret_cast(MOVQRIPRel), 2) != 0) - continue; - - auto &GOTBlock = E.getTarget().getBlock(); - assert(GOTBlock.getSize() == G.getPointerSize() && - "GOT entry block should be pointer sized"); - assert(GOTBlock.edges_size() == 1 && - "GOT entry should only have one outgoing edge"); - - auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); - JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); - - int64_t Displacement = TargetAddr - EdgeAddr + 4; - if (Displacement >= std::numeric_limits::min() && - Displacement <= std::numeric_limits::max()) { - // Change the edge kind as we don't go through GOT anymore. This is - // for formal correctness only. Technically, the two relocation kinds - // are resolved the same way. - E.setKind(x86_64::Delta32); - E.setTarget(GOTTarget); - E.setAddend(E.getAddend() - 4); - auto *BlockData = reinterpret_cast( - const_cast(B->getContent().data())); - BlockData[E.getOffset() - 2] = 0x8d; - LLVM_DEBUG({ - dbgs() << " Replaced GOT load wih LEA:\n "; - printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); - dbgs() << "\n"; - }); - } - } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) { - auto &StubBlock = E.getTarget().getBlock(); - assert( - StubBlock.getSize() == - sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) && - "Stub block should be stub sized"); - assert(StubBlock.edges_size() == 1 && - "Stub block should only have one outgoing edge"); - - auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); - assert(GOTBlock.getSize() == G.getPointerSize() && - "GOT block should be pointer sized"); - assert(GOTBlock.edges_size() == 1 && - "GOT block should only have one outgoing edge"); - - auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); - JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); - - int64_t Displacement = TargetAddr - EdgeAddr + 4; - if (Displacement >= std::numeric_limits::min() && - Displacement <= std::numeric_limits::max()) { - E.setKind(x86_64::BranchPCRel32); - E.setTarget(GOTTarget); - LLVM_DEBUG({ - dbgs() << " Replaced stub branch with direct branch:\n "; - printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); - dbgs() << "\n"; - }); - } - } - - return Error::success(); -} - static const char *getELFX86_64RelocName(uint32_t Type) { switch (Type) { #define ELF_RELOC(Name, Number) \ @@ -546,7 +470,7 @@ identifyELFSectionStartAndEndSymbols)); // Add GOT/Stubs optimizer pass. - Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); + Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -495,79 +495,6 @@ } // namespace -static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { - LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); - - for (auto *B : G.blocks()) - for (auto &E : B->edges()) - if (E.getKind() == x86_64::PCRel32GOTLoadRelaxable) { - assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); - - // Optimize GOT references. - auto &GOTBlock = E.getTarget().getBlock(); - assert(GOTBlock.getSize() == G.getPointerSize() && - "GOT entry block should be pointer sized"); - assert(GOTBlock.edges_size() == 1 && - "GOT entry should only have one outgoing edge"); - - auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); - JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); - - // Check that this is a recognized MOV instruction. - // FIXME: Can we assume this? - constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; - if (strncmp(B->getContent().data() + E.getOffset() - 3, - reinterpret_cast(MOVQRIPRel), 2) != 0) - continue; - - int64_t Displacement = TargetAddr - EdgeAddr + 4; - if (Displacement >= std::numeric_limits::min() && - Displacement <= std::numeric_limits::max()) { - E.setTarget(GOTTarget); - E.setKind(x86_64::Delta32); - E.setAddend(E.getAddend() - 4); - char *BlockData = B->getMutableContent(G).data(); - BlockData[E.getOffset() - 2] = (char)0x8d; - LLVM_DEBUG({ - dbgs() << " Replaced GOT load wih LEA:\n "; - printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); - dbgs() << "\n"; - }); - } - } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) { - auto &StubBlock = E.getTarget().getBlock(); - assert(StubBlock.getSize() == sizeof(x86_64::PointerJumpStubContent) && - "Stub block should be stub sized"); - assert(StubBlock.edges_size() == 1 && - "Stub block should only have one outgoing edge"); - - auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); - assert(GOTBlock.getSize() == G.getPointerSize() && - "GOT block should be pointer sized"); - assert(GOTBlock.edges_size() == 1 && - "GOT block should only have one outgoing edge"); - - auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); - JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); - - int64_t Displacement = TargetAddr - EdgeAddr + 4; - if (Displacement >= std::numeric_limits::min() && - Displacement <= std::numeric_limits::max()) { - E.setKind(x86_64::BranchPCRel32); - E.setTarget(GOTTarget); - LLVM_DEBUG({ - dbgs() << " Replaced stub branch with direct branch:\n "; - printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); - dbgs() << "\n"; - }); - } - } - - return Error::success(); -} - namespace llvm { namespace jitlink { @@ -615,7 +542,7 @@ PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::asPass); // Add GOT/Stubs optimizer pass. - Config.PreFixupPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); + Config.PreFixupPasses.push_back(x86_64::optimize_x86_64_GOTAndStubs); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -65,6 +65,80 @@ const char PointerJumpStubContent[6] = { static_cast(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00}; +Error optimize_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == x86_64::PCRel32GOTLoadRelaxable) { + // Replace GOT load with LEA only for MOVQ instructions. + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (E.getOffset() < 3 || + strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast(MOVQRIPRel), 2) != 0) + continue; + + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits::min() && + Displacement <= std::numeric_limits::max()) { + // Change the edge kind as we don't go through GOT anymore. This is + // for formal correctness only. Technically, the two relocation kinds + // are resolved the same way. + E.setKind(x86_64::Delta32); + E.setTarget(GOTTarget); + E.setAddend(E.getAddend() - 4); + auto *BlockData = reinterpret_cast( + const_cast(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, getGenericEdgeKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) { + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == sizeof(PointerJumpStubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits::min() && + Displacement <= std::numeric_limits::max()) { + E.setKind(x86_64::BranchPCRel32); + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, getGenericEdgeKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + } // end namespace x86_64 } // end namespace jitlink } // end namespace llvm