diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h @@ -52,6 +52,7 @@ TOCDelta16LO, TOCDelta16LODS, CallBranchDelta, + CallBranchDeltaRelaxable, // Need to restore r2 after the bl, suggesting the bl is followed by a nop. CallBranchDeltaRestoreTOC, // Request calling function with TOC. @@ -152,6 +153,29 @@ return G.addAnonymousSymbol(B, 0, StubInfo.Content.size(), true, false); } +inline Edge &findEdgeToFinalTargetFromStub(Block &StubEntryBlock) { + Symbol *GOTTarget = nullptr; + for (auto &E : StubEntryBlock.edges()) { + if (GOTTarget == nullptr) { + GOTTarget = &E.getTarget(); + continue; + } + assert(&E.getTarget() == GOTTarget && + "The stub entry is expected pointing to exact one target"); + } + assert(StubEntryBlock.edges_size() > 0 && + "The stub entry is expected to have at least one edge"); + Symbol &GOTEntry = StubEntryBlock.edges().begin()->getTarget(); + Block &GOTEntryBlock = GOTEntry.getBlock(); + assert(GOTEntryBlock.edges_size() == 1 && + "The GOT entry is expected pointing to exact one target"); + return *GOTEntryBlock.edges().begin(); +} + +inline Symbol &findFinalTargetFromStub(Block &StubEntryBlock) { + return findEdgeToFinalTargetFromStub(StubEntryBlock).getTarget(); +} + template class TOCTableManager : public TableManager> { public: @@ -210,12 +234,21 @@ E.setTarget(this->getEntryForTarget(G, E.getTarget())); // Addend to the stub is zero. E.setAddend(0); - } else + } else { // TODO: There are cases a local function call need a call stub. // 1. Caller uses TOC, the callee doesn't, need a r2 save stub. // 2. Caller doesn't use TOC, the callee does, need a r12 setup stub. - // 3. Branching target is out of range. - E.setKind(ppc64::CallBranchDelta); + // We always create a stub to perform local branching. If the branch + // target is within branching range, we can optimize that in a + // PreFixupPass. + this->StubKind = LongBranch; + E.setTarget(this->getEntryForTarget(G, E.getTarget())); + E.setKind(ppc64::CallBranchDeltaRelaxable); + Edge &TargetEdge = + findEdgeToFinalTargetFromStub(E.getTarget().getBlock()); + TargetEdge.setAddend(E.getAddend()); + E.setAddend(0); + } return true; } if (K == ppc64::RequestCallNoTOC) { @@ -483,6 +516,10 @@ return Error::success(); } +/// Optimize the GOT and Stub relocations if the edge target address is in +/// range. +Error optimizeGOTAndStubAccesses(LinkGraph &G); + } // end namespace llvm::jitlink::ppc64 #endif // LLVM_EXECUTIONENGINE_JITLINK_PPC64_H diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp @@ -491,6 +491,9 @@ Config.PostPrunePasses.push_back(buildTables_ELF_ppc64); + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(ppc64::optimizeGOTAndStubAccesses); + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp @@ -122,6 +122,8 @@ return "TOCDelta16LODS"; case CallBranchDelta: return "CallBranchDelta"; + case CallBranchDeltaRelaxable: + return "CallBranchDeltaRelaxable"; case CallBranchDeltaRestoreTOC: return "CallBranchDeltaRestoreTOC"; case RequestCall: @@ -137,4 +139,41 @@ } } +Error optimizeGOTAndStubAccesses(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) { + for (auto &E : B->edges()) { + const Edge::Kind K = E.getKind(); + if (K == ppc64::CallBranchDeltaRelaxable) { + Symbol &StubEntry = E.getTarget(); + Block &StubEntryBlock = StubEntry.getBlock(); + Edge &TargetEdge = findEdgeToFinalTargetFromStub(StubEntryBlock); + Symbol &Target = TargetEdge.getTarget(); + orc::ExecutorAddr TargetAddr = Target.getAddress(); + orc::ExecutorAddr EdgeAddr = B->getFixupAddress(E); + int64_t Displacement = TargetAddr - EdgeAddr; + if (LLVM_LIKELY(isInt<26>(Displacement))) { + E.setKind(CallBranchDelta); + E.setTarget(Target); + E.setAddend(TargetEdge.getAddend()); + LLVM_DEBUG({ + dbgs() << " Replaced branching to stub with direct branching:\n "; + printEdge(dbgs(), *B, E, getEdgeKindName(E.getKind())); + dbgs() << "\n"; + }); + } else { + E.setKind(CallBranchDelta); + LLVM_DEBUG({ + dbgs() << " Keep branching to stub:\n "; + printEdge(dbgs(), *B, E, getEdgeKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + } + } + return Error::success(); +} + } // end namespace llvm::jitlink::ppc64 diff --git a/llvm/test/ExecutionEngine/JITLink/ppc64/ppc64-local-branching.s b/llvm/test/ExecutionEngine/JITLink/ppc64/ppc64-local-branching.s new file mode 100644 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/ppc64/ppc64-local-branching.s @@ -0,0 +1,49 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc --triple=powerpc64le-unknown-linux-gnu --filetype=obj -o \ +# RUN: %t/elf_reloc.o %s +# RUN: not llvm-jitlink --noexec %t/elf_reloc.o 2>&1 | FileCheck %s +# RUN: llvm-mc --triple=powerpc64-unknown-linux-gnu --filetype=obj -o \ +# RUN: %t/elf_reloc.o %s +# RUN: not llvm-jitlink --noexec %t/elf_reloc.o 2>&1 | FileCheck %s +# CHECK: relocation target $__STUBS{{.*}} is out of range of CallBranchDelta fixup at {{.*}} + + .text + .abiversion 2 + .global main + .p2align 4 + .type main,@function +main: + addis 2, 12, .TOC.-main@ha + addi 2, 2, .TOC.-main@l + .localentry main, .-main + bl foo + li 3, 0 + blr + .size main, .-main + + .global foo + .p2align 4 + .type foo,@function +foo: + addis 2, 12, .TOC.-foo@ha + addi 2, 2, .TOC.-foo@l + .localentry foo, .-foo + li 3, 1 + bl bar + blr + .size foo, .-foo + +# Skip 32M so that bar is out of range of R_PPC64_REL24 in foo. + .skip 33554432 , 0 + + .global bar + .p2align 4 + .type bar,@function +bar: + addis 2, 12, .TOC.-bar@ha + addi 2, 2, .TOC.-bar@l + .localentry bar, .-bar + li 3, 2 + blr + .size bar, .-bar +