diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -437,7 +437,7 @@ virtual bool isUnsupportedBranch(const MCInst &Inst) const { return false; } /// Return true of the instruction is of pseudo kind. - bool isPseudo(const MCInst &Inst) const { + virtual bool isPseudo(const MCInst &Inst) const { return Info->get(Inst.getOpcode()).isPseudo(); } diff --git a/bolt/include/bolt/Passes/FixRISCVCallsPass.h b/bolt/include/bolt/Passes/FixRISCVCallsPass.h --- a/bolt/include/bolt/Passes/FixRISCVCallsPass.h +++ b/bolt/include/bolt/Passes/FixRISCVCallsPass.h @@ -6,9 +6,13 @@ // //===----------------------------------------------------------------------===// // -// This file declares the FixRISCVCallsPass class, which sets the JALR immediate -// to 0 for AUIPC/JALR pairs with a R_RISCV_CALL(_PLT) relocation. This is -// necessary since MC expects it to be zero in order to or-in fixups. +// This file declares the FixRISCVCallsPass class, which replaces all types of +// calls with PseudoCALL pseudo instructions. This ensures that relaxed calls +// get expanded to auipc/jalr pairs so that BOLT can freely reassign function +// addresses without having to worry about the limited range of relaxed calls. +// Using PseudoCALL also ensures that the RISC-V backend inserts the necessary +// relaxation-related relocations to allow JITLink to relax instruction back to +// shorter versions where possible. //===----------------------------------------------------------------------===// #ifndef BOLT_PASSES_FIXRISCVCALLSPASS_H diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -131,7 +131,7 @@ case llvm::Triple::riscv64: ArchName = "riscv64"; // RV64GC - FeaturesStr = "+m,+a,+f,+d,+zicsr,+zifencei,+c"; + FeaturesStr = "+m,+a,+f,+d,+zicsr,+zifencei,+c,+relax"; break; default: return createStringError(std::errc::not_supported, diff --git a/bolt/lib/Passes/FixRISCVCallsPass.cpp b/bolt/lib/Passes/FixRISCVCallsPass.cpp --- a/bolt/lib/Passes/FixRISCVCallsPass.cpp +++ b/bolt/lib/Passes/FixRISCVCallsPass.cpp @@ -10,23 +10,42 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) { auto &BC = BF.getBinaryContext(); + auto &MIB = BC.MIB; + auto *Ctx = BC.Ctx.get(); for (auto &BB : BF) { - for (auto II = BB.begin(), IE = BB.end(); II != IE; ++II) { + for (auto II = BB.begin(); II != BB.end();) { + if (MIB->isCall(*II) && !MIB->isIndirectCall(*II)) { + auto *Target = MIB->getTargetSymbol(*II); + assert(Target && "Cannot find call target"); + + auto L = BC.scopeLock(); + + if (MIB->isTailCall(*II)) + MIB->createTailCall(*II, Target, Ctx); + else + MIB->createCall(*II, Target, Ctx); + + ++II; + continue; + } + auto NextII = std::next(II); - if (NextII == IE) + if (NextII == BB.end()) break; - if (!BC.MIB->isRISCVCall(*II, *NextII)) - continue; + if (MIB->isRISCVCall(*II, *NextII)) { + auto *Target = MIB->getTargetSymbol(*II); + assert(Target && "Cannot find call target"); - auto L = BC.scopeLock(); + auto L = BC.scopeLock(); + MIB->createCall(*II, Target, Ctx); + II = BB.eraseInstruction(NextII); + continue; + } - // The MC layer handles R_RISCV_CALL_PLT but assumes that the immediate - // in the JALR is zero (fixups are or'ed into instructions). Note that - // NextII is guaranteed to point to a JALR by isRISCVCall. - NextII->getOperand(2).setImm(0); + ++II; } } } diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp --- a/bolt/lib/Rewrite/JITLinkLinker.cpp +++ b/bolt/lib/Rewrite/JITLinkLinker.cpp @@ -8,6 +8,7 @@ #include "bolt/Rewrite/JITLinkLinker.h" #include "bolt/Core/BinaryData.h" #include "bolt/Rewrite/RewriteInstance.h" +#include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" @@ -93,6 +94,18 @@ Error modifyPassConfig(jitlink::LinkGraph &G, jitlink::PassConfiguration &Config) override { Config.PrePrunePasses.push_back(markSectionsLive); + Config.PostAllocationPasses.push_back([this](auto &G) { + MapSections([&G](const BinarySection &Section, uint64_t Address) { + reassignSectionAddress(G, Section, Address); + }); + return Error::success(); + }); + + if (G.getTargetTriple().isRISCV()) { + Config.PostAllocationPasses.push_back( + jitlink::createRelaxationPass_ELF_riscv()); + } + return Error::success(); } @@ -137,10 +150,6 @@ } Error notifyResolved(jitlink::LinkGraph &G) override { - MapSections([&G](const BinarySection &Section, uint64_t Address) { - reassignSectionAddress(G, Section, Address); - }); - for (auto *Symbol : G.defined_symbols()) { SymbolInfo Info{Symbol->getAddress().getValue(), Symbol->getSize()}; Linker.Symtab.insert({Symbol->getName().str(), Info}); diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -64,6 +64,29 @@ return isNop(Inst) || isCNop(Inst); } + bool isPseudo(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + default: + return MCPlusBuilder::isPseudo(Inst); + case RISCV::PseudoCALL: + case RISCV::PseudoTAIL: + return false; + } + } + + bool isIndirectCall(const MCInst &Inst) const override { + if (!isCall(Inst)) + return false; + + switch (Inst.getOpcode()) { + default: + return false; + case RISCV::JALR: + case RISCV::C_JALR: + return true; + } + } + bool hasPCRelOperand(const MCInst &Inst) const override { switch (Inst.getOpcode()) { default: @@ -176,6 +199,26 @@ return StringRef("\0\0\0\0", 4); } + bool createCall(unsigned Opcode, MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) { + Inst.setOpcode(Opcode); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr(RISCVMCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + RISCVMCExpr::VK_RISCV_CALL, *Ctx))); + return true; + } + + bool createCall(MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) override { + return createCall(RISCV::PseudoCALL, Inst, Target, Ctx); + } + + bool createTailCall(MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) override { + return createCall(RISCV::PseudoTAIL, Inst, Target, Ctx); + } + bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, const MCSymbol *&TBB, const MCSymbol *&FBB, MCInst *&CondBranch, @@ -235,6 +278,7 @@ case RISCV::C_J: OpNum = 0; return true; + case RISCV::AUIPC: case RISCV::JAL: case RISCV::C_BEQZ: case RISCV::C_BNEZ: @@ -276,7 +320,7 @@ if (!Op.isExpr()) return nullptr; - return MCPlusBuilder::getTargetSymbol(Op.getExpr()); + return getTargetSymbol(Op.getExpr()); } bool lowerTailCall(MCInst &Inst) override { diff --git a/bolt/test/RISCV/internal-func-reloc.s b/bolt/test/RISCV/internal-func-reloc.s --- a/bolt/test/RISCV/internal-func-reloc.s +++ b/bolt/test/RISCV/internal-func-reloc.s @@ -26,7 +26,7 @@ .p2align 1 // CHECK: : // CHECK-NEXT: auipc a0, 0 -// CHECK-NEXT: addi a0, a0, 12 +// CHECK-NEXT: addi a0, a0, 64 f: nop 1: diff --git a/bolt/test/RISCV/lit.local.cfg b/bolt/test/RISCV/lit.local.cfg --- a/bolt/test/RISCV/lit.local.cfg +++ b/bolt/test/RISCV/lit.local.cfg @@ -1,7 +1,7 @@ if 'RISCV' not in config.root.targets: config.unsupported = True -flags = '--target=riscv64 -nostdlib -ffreestanding -Wl,--no-relax,--emit-relocs' +flags = '--target=riscv64 -nostdlib -ffreestanding -Wl,--emit-relocs' config.substitutions.insert(0, ('%cflags', f'%cflags {flags}')) config.substitutions.insert(0, ('%cxxflags', f'%cxxflags {flags}')) diff --git a/bolt/test/RISCV/relax.s b/bolt/test/RISCV/relax.s new file mode 100644 --- /dev/null +++ b/bolt/test/RISCV/relax.s @@ -0,0 +1,47 @@ +// RUN: llvm-mc -triple riscv64 -mattr=+c,+relax -filetype obj -o %t.o %s +// RUN: ld.lld --emit-relocs -o %t %t.o +// RUN: llvm-bolt --print-cfg --print-fix-riscv-calls --print-only=_start \ +// RUN: -o %t.bolt %t | FileCheck %s +// RUN: llvm-objdump -d %t.bolt | FileCheck --check-prefix=OBJDUMP %s + +// CHECK: Binary Function "_start" after building cfg { +// CHECK: jal ra, near_f +// CHECK-NEXT: auipc ra, far_f@plt +// CHECK-NEXT: jalr ra, 12(ra) +// CHECK-NEXT: j near_f + +// CHECK: Binary Function "_start" after fix-riscv-calls { +// CHECK: call near_f +// CHECK-NEXT: call far_f +// CHECK-NEXT: tail near_f + +// OBJDUMP: 0000000000600000 <_start>: +// OBJDUMP-NEXT: jal 0x600040 +// OBJDUMP-NEXT: auipc ra, 512 +// OBJDUMP-NEXT: jalr 124(ra) +// OBJDUMP-NEXT: j 0x600040 +// OBJDUMP: 0000000000600040 : +// OBJDUMP: 0000000000800080 : + + .text + .globl _start + .p2align 1 +_start: + call near_f + call far_f + tail near_f + .size _start, .-_start + + .global near_f + .p2align 1 +near_f: + ret + .size near_f, .-near_f + + .skip (1 << 21) + + .global far_f + .p2align 1 +far_f: + ret + .size far_f, .-far_f diff --git a/bolt/test/RISCV/reloc-call.s b/bolt/test/RISCV/reloc-call.s --- a/bolt/test/RISCV/reloc-call.s +++ b/bolt/test/RISCV/reloc-call.s @@ -14,8 +14,7 @@ .globl _start .p2align 1 _start: -// CHECK: auipc ra, f -// CHECK-NEXT: jalr ra +// CHECK: call f call f ret .size _start, .-_start