diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h @@ -52,6 +52,7 @@ TOCDelta16LO, TOCDelta16LODS, CallBranchDelta, + CallBranchDeltaRelaxable, // Need to restore r2 after the bl, suggesting the bl is followed by a nop. CallBranchDeltaRestoreTOC, // Request calling function with TOC. @@ -152,6 +153,29 @@ return G.addAnonymousSymbol(B, 0, StubInfo.Content.size(), true, false); } +inline Edge &findEdgeToFinalTargetFromStub(Block &StubEntryBlock) { + Symbol *GOTTarget = nullptr; + for (auto &E : StubEntryBlock.edges()) { + if (GOTTarget == nullptr) { + GOTTarget = &E.getTarget(); + continue; + } + assert(&E.getTarget() == GOTTarget && + "The stub entry is expected pointing to exact one target"); + } + assert(StubEntryBlock.edges_size() > 0 && + "The stub entry is expected to have at least one edge"); + Symbol &GOTEntry = StubEntryBlock.edges().begin()->getTarget(); + Block &GOTEntryBlock = GOTEntry.getBlock(); + assert(GOTEntryBlock.edges_size() == 1 && + "The GOT entry is expected pointing to exact one target"); + return *GOTEntryBlock.edges().begin(); +} + +inline Symbol &findFinalTargetFromStub(Block &StubEntryBlock) { + return findEdgeToFinalTargetFromStub(StubEntryBlock).getTarget(); +} + template class TOCTableManager : public TableManager> { public: @@ -210,12 +234,21 @@ E.setTarget(this->getEntryForTarget(G, E.getTarget())); // Addend to the stub is zero. E.setAddend(0); - } else + } else { // TODO: There are cases a local function call need a call stub. // 1. Caller uses TOC, the callee doesn't, need a r2 save stub. // 2. Caller doesn't use TOC, the callee does, need a r12 setup stub. - // 3. Branching target is out of range. - E.setKind(ppc64::CallBranchDelta); + // We always create a stub to perform local branching. If the branch + // target is within branching range, we can optimize that in a + // PreFixupPass. + this->StubKind = LongBranch; + E.setTarget(this->getEntryForTarget(G, E.getTarget())); + E.setKind(ppc64::CallBranchDeltaRelaxable); + Edge &TargetEdge = + findEdgeToFinalTargetFromStub(E.getTarget().getBlock()); + TargetEdge.setAddend(E.getAddend()); + E.setAddend(0); + } return true; } if (K == ppc64::RequestCallNoTOC) { @@ -483,6 +516,10 @@ return Error::success(); } +/// Optimize the GOT and Stub relocations if the edge target address is in +/// range. +Error optimizeGOTAndStubAccesses(LinkGraph &G); + } // end namespace llvm::jitlink::ppc64 #endif // LLVM_EXECUTIONENGINE_JITLINK_PPC64_H diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp @@ -491,6 +491,9 @@ Config.PostPrunePasses.push_back(buildTables_ELF_ppc64); + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(ppc64::optimizeGOTAndStubAccesses); + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp @@ -122,6 +122,8 @@ return "TOCDelta16LODS"; case CallBranchDelta: return "CallBranchDelta"; + case CallBranchDeltaRelaxable: + return "CallBranchDeltaRelaxable"; case CallBranchDeltaRestoreTOC: return "CallBranchDeltaRestoreTOC"; case RequestCall: @@ -137,4 +139,41 @@ } } +Error optimizeGOTAndStubAccesses(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) { + for (auto &E : B->edges()) { + Edge::Kind K = E.getKind(); + if (K == ppc64::CallBranchDeltaRelaxable) { + Symbol &StubEntry = E.getTarget(); + Block &StubEntryBlock = StubEntry.getBlock(); + Edge &TargetEdge = findEdgeToFinalTargetFromStub(StubEntryBlock); + Symbol &Target = TargetEdge.getTarget(); + orc::ExecutorAddr TargetAddr = Target.getAddress(); + orc::ExecutorAddr EdgeAddr = B->getFixupAddress(E); + int64_t Displacement = TargetAddr - EdgeAddr; + if (LLVM_LIKELY(isInt<26>(Displacement))) { + E.setKind(CallBranchDelta); + E.setTarget(Target); + E.setAddend(TargetEdge.getAddend()); + LLVM_DEBUG({ + dbgs() << " Replaced branching to stub with direct branching:\n"; + printEdge(dbgs(), *B, E, getEdgeKindName(K)); + dbgs() << "\n"; + }); + } else { + E.setKind(CallBranchDelta); + LLVM_DEBUG({ + dbgs() << " Keep branching to stub:\n"; + printEdge(dbgs(), *B, E, getEdgeKindName(K)); + dbgs() << "\n"; + }); + } + } + } + } + return Error::success(); +} + } // end namespace llvm::jitlink::ppc64