diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp index 2e7b20d46cb0..06dd863f31b2 100644 --- a/lld/ELF/Arch/PPC64.cpp +++ b/lld/ELF/Arch/PPC64.cpp @@ -1,1555 +1,1503 @@ //===- PPC64.cpp ----------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" #include "Thunks.h" #include "lld/Common/ErrorHandler.h" #include "lld/Common/Memory.h" #include "llvm/Support/Endian.h" using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; using namespace lld; using namespace lld::elf; constexpr uint64_t ppc64TocOffset = 0x8000; constexpr uint64_t dynamicThreadPointerOffset = 0x8000; // The instruction encoding of bits 21-30 from the ISA for the Xform and Dform // instructions that can be used as part of the initial exec TLS sequence. enum XFormOpcd { LBZX = 87, LHZX = 279, LWZX = 23, LDX = 21, STBX = 215, STHX = 407, STWX = 151, STDX = 149, ADD = 266, }; enum DFormOpcd { LBZ = 34, LBZU = 35, LHZ = 40, LHZU = 41, LHAU = 43, LWZ = 32, LWZU = 33, LFSU = 49, LD = 58, LFDU = 51, STB = 38, STBU = 39, STH = 44, STHU = 45, STW = 36, STWU = 37, STFSU = 53, STFDU = 55, STD = 62, ADDI = 14 }; constexpr uint32_t NOP = 0x60000000; enum class PPCLegacyInsn : uint32_t { NOINSN = 0, // Loads. LBZ = 0x88000000, LHZ = 0xa0000000, LWZ = 0x80000000, LHA = 0xa8000000, LWA = 0xe8000002, LD = 0xe8000000, LFS = 0xC0000000, LXSSP = 0xe4000003, LFD = 0xc8000000, LXSD = 0xe4000002, LXV = 0xf4000001, LXVP = 0x18000000, // Stores. STB = 0x98000000, STH = 0xb0000000, STW = 0x90000000, STD = 0xf8000000, STFS = 0xd0000000, STXSSP = 0xf4000003, STFD = 0xd8000000, STXSD = 0xf4000002, STXV = 0xf4000005, STXVP = 0x18000001 }; enum class PPCPrefixedInsn : uint64_t { NOINSN = 0, PREFIX_MLS = 0x0610000000000000, PREFIX_8LS = 0x0410000000000000, // Loads. PLBZ = PREFIX_MLS, PLHZ = PREFIX_MLS, PLWZ = PREFIX_MLS, PLHA = PREFIX_MLS, PLWA = PREFIX_8LS | 0xa4000000, PLD = PREFIX_8LS | 0xe4000000, PLFS = PREFIX_MLS, PLXSSP = PREFIX_8LS | 0xac000000, PLFD = PREFIX_MLS, PLXSD = PREFIX_8LS | 0xa8000000, PLXV = PREFIX_8LS | 0xc8000000, PLXVP = PREFIX_8LS | 0xe8000000, // Stores. PSTB = PREFIX_MLS, PSTH = PREFIX_MLS, PSTW = PREFIX_MLS, PSTD = PREFIX_8LS | 0xf4000000, PSTFS = PREFIX_MLS, PSTXSSP = PREFIX_8LS | 0xbc000000, PSTFD = PREFIX_MLS, PSTXSD = PREFIX_8LS | 0xb8000000, PSTXV = PREFIX_8LS | 0xd8000000, PSTXVP = PREFIX_8LS | 0xf8000000 }; static bool checkPPCLegacyInsn(uint32_t encoding) { PPCLegacyInsn insn = static_cast(encoding); if (insn == PPCLegacyInsn::NOINSN) return false; #define PCREL_OPT(Legacy, PCRel, InsnMask) \ if (insn == PPCLegacyInsn::Legacy) \ return true; #include "PPCInsns.def" #undef PCREL_OPT return false; } // Masks to apply to legacy instructions when converting them to prefixed, // pc-relative versions. For the most part, the primary opcode is shared // between the legacy instruction and the suffix of its prefixed version. // However, there are some instances where that isn't the case (DS-Form and // DQ-form instructions). enum class LegacyToPrefixMask : uint64_t { NOMASK = 0x0, OPC_AND_RST = 0xffe00000, // Primary opc (0-5) and R[ST] (6-10). ONLY_RST = 0x3e00000, // [RS]T (6-10). ST_STX28_TO5 = 0x8000000003e00000, // S/T (6-10) - The [S/T]X bit moves from 28 to 5. }; uint64_t elf::getPPC64TocBase() { // The TOC consists of sections .got, .toc, .tocbss, .plt in that order. The // TOC starts where the first of these sections starts. We always create a // .got when we see a relocation that uses it, so for us the start is always // the .got. uint64_t tocVA = in.got->getVA(); // Per the ppc64-elf-linux ABI, The TOC base is TOC value plus 0x8000 // thus permitting a full 64 Kbytes segment. Note that the glibc startup // code (crt1.o) assumes that you can get from the TOC base to the // start of the .toc section with only a single (signed) 16-bit relocation. return tocVA + ppc64TocOffset; } unsigned elf::getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther) { // The offset is encoded into the 3 most significant bits of the st_other // field, with some special values described in section 3.4.1 of the ABI: // 0 --> Zero offset between the GEP and LEP, and the function does NOT use // the TOC pointer (r2). r2 will hold the same value on returning from // the function as it did on entering the function. // 1 --> Zero offset between the GEP and LEP, and r2 should be treated as a // caller-saved register for all callers. // 2-6 --> The binary logarithm of the offset eg: // 2 --> 2^2 = 4 bytes --> 1 instruction. // 6 --> 2^6 = 64 bytes --> 16 instructions. // 7 --> Reserved. uint8_t gepToLep = (stOther >> 5) & 7; if (gepToLep < 2) return 0; // The value encoded in the st_other bits is the // log-base-2(offset). if (gepToLep < 7) return 1 << gepToLep; error("reserved value of 7 in the 3 most-significant-bits of st_other"); return 0; } bool elf::isPPC64SmallCodeModelTocReloc(RelType type) { // The only small code model relocations that access the .toc section. return type == R_PPC64_TOC16 || type == R_PPC64_TOC16_DS; } void elf::writePrefixedInstruction(uint8_t *loc, uint64_t insn) { insn = config->isLE ? insn << 32 | insn >> 32 : insn; write64(loc, insn); } static bool addOptional(StringRef name, uint64_t value, std::vector &defined) { Symbol *sym = symtab->find(name); if (!sym || sym->isDefined()) return false; sym->resolve(Defined{/*file=*/nullptr, saver.save(name), STB_GLOBAL, STV_HIDDEN, STT_FUNC, value, /*size=*/0, /*section=*/nullptr}); defined.push_back(cast(sym)); return true; } // If from is 14, write ${prefix}14: firstInsn; ${prefix}15: // firstInsn+0x200008; ...; ${prefix}31: firstInsn+(31-14)*0x200008; $tail // The labels are defined only if they exist in the symbol table. static void writeSequence(MutableArrayRef buf, const char *prefix, int from, uint32_t firstInsn, ArrayRef tail) { std::vector defined; char name[16]; int first; uint32_t *ptr = buf.data(); for (int r = from; r < 32; ++r) { format("%s%d", prefix, r).snprint(name, sizeof(name)); if (addOptional(name, 4 * (r - from), defined) && defined.size() == 1) first = r - from; write32(ptr++, firstInsn + 0x200008 * (r - from)); } for (uint32_t insn : tail) write32(ptr++, insn); assert(ptr == &*buf.end()); if (defined.empty()) return; // The full section content has the extent of [begin, end). We drop unused // instructions and write [first,end). auto *sec = make( nullptr, SHF_ALLOC, SHT_PROGBITS, 4, makeArrayRef(reinterpret_cast(buf.data() + first), 4 * (buf.size() - first)), ".text"); inputSections.push_back(sec); for (Defined *sym : defined) { sym->section = sec; sym->value -= 4 * first; } } // Implements some save and restore functions as described by ELF V2 ABI to be // compatible with GCC. With GCC -Os, when the number of call-saved registers // exceeds a certain threshold, GCC generates _savegpr0_* _restgpr0_* calls and // expects the linker to define them. See // https://sourceware.org/pipermail/binutils/2002-February/017444.html and // https://sourceware.org/pipermail/binutils/2004-August/036765.html . This is // weird because libgcc.a would be the natural place. The linker generation // approach has the advantage that the linker can generate multiple copies to // avoid long branch thunks. However, we don't consider the advantage // significant enough to complicate our trunk implementation, so we take the // simple approach and synthesize .text sections providing the implementation. void elf::addPPC64SaveRestore() { static uint32_t savegpr0[20], restgpr0[21], savegpr1[19], restgpr1[19]; constexpr uint32_t blr = 0x4e800020, mtlr_0 = 0x7c0803a6; // _restgpr0_14: ld 14, -144(1); _restgpr0_15: ld 15, -136(1); ... // Tail: ld 0, 16(1); mtlr 0; blr writeSequence(restgpr0, "_restgpr0_", 14, 0xe9c1ff70, {0xe8010010, mtlr_0, blr}); // _restgpr1_14: ld 14, -144(12); _restgpr1_15: ld 15, -136(12); ... // Tail: blr writeSequence(restgpr1, "_restgpr1_", 14, 0xe9ccff70, {blr}); // _savegpr0_14: std 14, -144(1); _savegpr0_15: std 15, -136(1); ... // Tail: std 0, 16(1); blr writeSequence(savegpr0, "_savegpr0_", 14, 0xf9c1ff70, {0xf8010010, blr}); // _savegpr1_14: std 14, -144(12); _savegpr1_15: std 15, -136(12); ... // Tail: blr writeSequence(savegpr1, "_savegpr1_", 14, 0xf9ccff70, {blr}); } // Find the R_PPC64_ADDR64 in .rela.toc with matching offset. template static std::pair getRelaTocSymAndAddend(InputSectionBase *tocSec, uint64_t offset) { if (tocSec->numRelocations == 0) return {}; // .rela.toc contains exclusively R_PPC64_ADDR64 relocations sorted by // r_offset: 0, 8, 16, etc. For a given Offset, Offset / 8 gives us the // relocation index in most cases. // // In rare cases a TOC entry may store a constant that doesn't need an // R_PPC64_ADDR64, the corresponding r_offset is therefore missing. Offset / 8 // points to a relocation with larger r_offset. Do a linear probe then. // Constants are extremely uncommon in .toc and the extra number of array // accesses can be seen as a small constant. ArrayRef relas = tocSec->template relas(); uint64_t index = std::min(offset / 8, relas.size() - 1); for (;;) { if (relas[index].r_offset == offset) { Symbol &sym = tocSec->getFile()->getRelocTargetSym(relas[index]); return {dyn_cast(&sym), getAddend(relas[index])}; } if (relas[index].r_offset < offset || index == 0) break; --index; } return {}; } // When accessing a symbol defined in another translation unit, compilers // reserve a .toc entry, allocate a local label and generate toc-indirect // instructions: // // addis 3, 2, .LC0@toc@ha # R_PPC64_TOC16_HA // ld 3, .LC0@toc@l(3) # R_PPC64_TOC16_LO_DS, load the address from a .toc entry // ld/lwa 3, 0(3) # load the value from the address // // .section .toc,"aw",@progbits // .LC0: .tc var[TC],var // // If var is defined, non-preemptable and addressable with a 32-bit signed // offset from the toc base, the address of var can be computed by adding an // offset to the toc base, saving a load. // // addis 3,2,var@toc@ha # this may be relaxed to a nop, // addi 3,3,var@toc@l # then this becomes addi 3,2,var@toc // ld/lwa 3, 0(3) # load the value from the address // // Returns true if the relaxation is performed. bool elf::tryRelaxPPC64TocIndirection(const Relocation &rel, uint8_t *bufLoc) { assert(config->tocOptimize); if (rel.addend < 0) return false; // If the symbol is not the .toc section, this isn't a toc-indirection. Defined *defSym = dyn_cast(rel.sym); if (!defSym || !defSym->isSection() || defSym->section->name != ".toc") return false; Defined *d; int64_t addend; auto *tocISB = cast(defSym->section); std::tie(d, addend) = config->isLE ? getRelaTocSymAndAddend(tocISB, rel.addend) : getRelaTocSymAndAddend(tocISB, rel.addend); // Only non-preemptable defined symbols can be relaxed. if (!d || d->isPreemptible) return false; // R_PPC64_ADDR64 should have created a canonical PLT for the non-preemptable // ifunc and changed its type to STT_FUNC. assert(!d->isGnuIFunc()); // Two instructions can materialize a 32-bit signed offset from the toc base. uint64_t tocRelative = d->getVA(addend) - getPPC64TocBase(); if (!isInt<32>(tocRelative)) return false; // Add PPC64TocOffset that will be subtracted by PPC64::relocate(). target->relaxGot(bufLoc, rel, tocRelative + ppc64TocOffset); return true; } namespace { class PPC64 final : public TargetInfo { public: PPC64(); int getTlsGdRelaxSkip(RelType type) const override; uint32_t calcEFlags() const override; RelExpr getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const override; RelType getDynRel(RelType type) const override; void writePltHeader(uint8_t *buf) const override; void writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const override; void writeIplt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const override; void relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void writeGotHeader(uint8_t *buf) const override; bool needsThunk(RelExpr expr, RelType type, const InputFile *file, uint64_t branchAddr, const Symbol &s, int64_t a) const override; uint32_t getThunkSectionSpacing() const override; bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; RelExpr adjustRelaxExpr(RelType type, const uint8_t *data, RelExpr expr) const override; void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const override; void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const override; bool adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, uint8_t stOther) const override; }; } // namespace // Relocation masks following the #lo(value), #hi(value), #ha(value), // #higher(value), #highera(value), #highest(value), and #highesta(value) // macros defined in section 4.5.1. Relocation Types of the PPC-elf64abi // document. static uint16_t lo(uint64_t v) { return v; } static uint16_t hi(uint64_t v) { return v >> 16; } static uint16_t ha(uint64_t v) { return (v + 0x8000) >> 16; } static uint16_t higher(uint64_t v) { return v >> 32; } static uint16_t highera(uint64_t v) { return (v + 0x8000) >> 32; } static uint16_t highest(uint64_t v) { return v >> 48; } static uint16_t highesta(uint64_t v) { return (v + 0x8000) >> 48; } // Extracts the 'PO' field of an instruction encoding. static uint8_t getPrimaryOpCode(uint32_t encoding) { return (encoding >> 26); } static bool isDQFormInstruction(uint32_t encoding) { switch (getPrimaryOpCode(encoding)) { default: return false; case 6: // Power10 paired loads/stores (lxvp, stxvp). case 56: // The only instruction with a primary opcode of 56 is `lq`. return true; case 61: // There are both DS and DQ instruction forms with this primary opcode. // Namely `lxv` and `stxv` are the DQ-forms that use it. // The DS 'XO' bits being set to 01 is restricted to DQ form. return (encoding & 3) == 0x1; } } static bool isDSFormInstruction(PPCLegacyInsn insn) { switch (insn) { default: return false; case PPCLegacyInsn::LWA: case PPCLegacyInsn::LD: case PPCLegacyInsn::LXSD: case PPCLegacyInsn::LXSSP: case PPCLegacyInsn::STD: case PPCLegacyInsn::STXSD: case PPCLegacyInsn::STXSSP: return true; } } static PPCLegacyInsn getPPCLegacyInsn(uint32_t encoding) { uint32_t opc = encoding & 0xfc000000; // If the primary opcode is shared between multiple instructions, we need to // fix it up to match the actual instruction we are after. if ((opc == 0xe4000000 || opc == 0xe8000000 || opc == 0xf4000000 || opc == 0xf8000000) && !isDQFormInstruction(encoding)) opc = encoding & 0xfc000003; else if (opc == 0xf4000000) opc = encoding & 0xfc000007; else if (opc == 0x18000000) opc = encoding & 0xfc00000f; // If the value is not one of the enumerators in PPCLegacyInsn, we want to // return PPCLegacyInsn::NOINSN. if (!checkPPCLegacyInsn(opc)) return PPCLegacyInsn::NOINSN; return static_cast(opc); } static PPCPrefixedInsn getPCRelativeForm(PPCLegacyInsn insn) { switch (insn) { #define PCREL_OPT(Legacy, PCRel, InsnMask) \ case PPCLegacyInsn::Legacy: \ return PPCPrefixedInsn::PCRel #include "PPCInsns.def" #undef PCREL_OPT } return PPCPrefixedInsn::NOINSN; } static LegacyToPrefixMask getInsnMask(PPCLegacyInsn insn) { switch (insn) { #define PCREL_OPT(Legacy, PCRel, InsnMask) \ case PPCLegacyInsn::Legacy: \ return LegacyToPrefixMask::InsnMask #include "PPCInsns.def" #undef PCREL_OPT } return LegacyToPrefixMask::NOMASK; } static uint64_t getPCRelativeForm(uint32_t encoding) { PPCLegacyInsn origInsn = getPPCLegacyInsn(encoding); PPCPrefixedInsn pcrelInsn = getPCRelativeForm(origInsn); if (pcrelInsn == PPCPrefixedInsn::NOINSN) return UINT64_C(-1); LegacyToPrefixMask origInsnMask = getInsnMask(origInsn); uint64_t pcrelEncoding = (uint64_t)pcrelInsn | (encoding & (uint64_t)origInsnMask); // If the mask requires moving bit 28 to bit 5, do that now. if (origInsnMask == LegacyToPrefixMask::ST_STX28_TO5) pcrelEncoding |= (encoding & 0x8) << 23; return pcrelEncoding; } static bool isInstructionUpdateForm(uint32_t encoding) { switch (getPrimaryOpCode(encoding)) { default: return false; case LBZU: case LHAU: case LHZU: case LWZU: case LFSU: case LFDU: case STBU: case STHU: case STWU: case STFSU: case STFDU: return true; // LWA has the same opcode as LD, and the DS bits is what differentiates // between LD/LDU/LWA case LD: case STD: return (encoding & 3) == 1; } } // Compute the total displacement between the prefixed instruction that gets // to the start of the data and the load/store instruction that has the offset // into the data structure. // For example: // paddi 3, 0, 1000, 1 // lwz 3, 20(3) // Should add up to 1020 for total displacement. static int64_t getTotalDisp(uint64_t prefixedInsn, uint32_t accessInsn) { int64_t disp34 = llvm::SignExtend64( ((prefixedInsn & 0x3ffff00000000) >> 16) | (prefixedInsn & 0xffff), 34); int32_t disp16 = llvm::SignExtend32(accessInsn & 0xffff, 16); // For DS and DQ form instructions, we need to mask out the XO bits. if (isDQFormInstruction(accessInsn)) disp16 &= ~0xf; else if (isDSFormInstruction(getPPCLegacyInsn(accessInsn))) disp16 &= ~0x3; return disp34 + disp16; } // There are a number of places when we either want to read or write an // instruction when handling a half16 relocation type. On big-endian the buffer // pointer is pointing into the middle of the word we want to extract, and on // little-endian it is pointing to the start of the word. These 2 helpers are to // simplify reading and writing in that context. static void writeFromHalf16(uint8_t *loc, uint32_t insn) { write32(config->isLE ? loc : loc - 2, insn); } static uint32_t readFromHalf16(const uint8_t *loc) { return read32(config->isLE ? loc : loc - 2); } static uint64_t readPrefixedInstruction(const uint8_t *loc) { uint64_t fullInstr = read64(loc); return config->isLE ? (fullInstr << 32 | fullInstr >> 32) : fullInstr; } PPC64::PPC64() { copyRel = R_PPC64_COPY; gotRel = R_PPC64_GLOB_DAT; noneRel = R_PPC64_NONE; pltRel = R_PPC64_JMP_SLOT; relativeRel = R_PPC64_RELATIVE; iRelativeRel = R_PPC64_IRELATIVE; symbolicRel = R_PPC64_ADDR64; pltHeaderSize = 60; pltEntrySize = 4; ipltEntrySize = 16; // PPC64PltCallStub::size gotBaseSymInGotPlt = false; gotHeaderEntriesNum = 1; gotPltHeaderEntriesNum = 2; needsThunks = true; tlsModuleIndexRel = R_PPC64_DTPMOD64; tlsOffsetRel = R_PPC64_DTPREL64; tlsGotRel = R_PPC64_TPREL64; needsMoreStackNonSplit = false; // We need 64K pages (at least under glibc/Linux, the loader won't // set different permissions on a finer granularity than that). defaultMaxPageSize = 65536; // The PPC64 ELF ABI v1 spec, says: // // It is normally desirable to put segments with different characteristics // in separate 256 Mbyte portions of the address space, to give the // operating system full paging flexibility in the 64-bit address space. // // And because the lowest non-zero 256M boundary is 0x10000000, PPC64 linkers // use 0x10000000 as the starting address. defaultImageBase = 0x10000000; write32(trapInstr.data(), 0x7fe00008); } int PPC64::getTlsGdRelaxSkip(RelType type) const { // A __tls_get_addr call instruction is marked with 2 relocations: // // R_PPC64_TLSGD / R_PPC64_TLSLD: marker relocation // R_PPC64_REL24: __tls_get_addr // // After the relaxation we no longer call __tls_get_addr and should skip both // relocations to not create a false dependence on __tls_get_addr being // defined. if (type == R_PPC64_TLSGD || type == R_PPC64_TLSLD) return 2; return 1; } static uint32_t getEFlags(InputFile *file) { if (config->ekind == ELF64BEKind) return cast>(file)->getObj().getHeader().e_flags; return cast>(file)->getObj().getHeader().e_flags; } // This file implements v2 ABI. This function makes sure that all // object files have v2 or an unspecified version as an ABI version. uint32_t PPC64::calcEFlags() const { for (InputFile *f : objectFiles) { uint32_t flag = getEFlags(f); if (flag == 1) error(toString(f) + ": ABI version 1 is not supported"); else if (flag > 2) error(toString(f) + ": unrecognized e_flags: " + Twine(flag)); } return 2; } void PPC64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const { switch (rel.type) { case R_PPC64_TOC16_HA: // Convert "addis reg, 2, .LC0@toc@h" to "addis reg, 2, var@toc@h" or "nop". relocate(loc, rel, val); break; case R_PPC64_TOC16_LO_DS: { // Convert "ld reg, .LC0@toc@l(reg)" to "addi reg, reg, var@toc@l" or // "addi reg, 2, var@toc". uint32_t insn = readFromHalf16(loc); if (getPrimaryOpCode(insn) != LD) error("expected a 'ld' for got-indirect to toc-relative relaxing"); writeFromHalf16(loc, (insn & 0x03ffffff) | 0x38000000); relocateNoSym(loc, R_PPC64_TOC16_LO, val); break; } case R_PPC64_GOT_PCREL34: { // Clear the first 8 bits of the prefix and the first 6 bits of the // instruction (the primary opcode). uint64_t insn = readPrefixedInstruction(loc); if ((insn & 0xfc000000) != 0xe4000000) error("expected a 'pld' for got-indirect to pc-relative relaxing"); insn &= ~0xff000000fc000000; // Replace the cleared bits with the values for PADDI (0x600000038000000); insn |= 0x600000038000000; writePrefixedInstruction(loc, insn); relocate(loc, rel, val); break; } case R_PPC64_PCREL_OPT: { // We can only relax this if the R_PPC64_GOT_PCREL34 at this offset can // be relaxed. The eligibility for the relaxation needs to be determined // on that relocation since this one does not relocate a symbol. uint64_t insn = readPrefixedInstruction(loc); uint32_t accessInsn = read32(loc + rel.addend); uint64_t pcRelInsn = getPCRelativeForm(accessInsn); // This error is not necessary for correctness but is emitted for now // to ensure we don't miss these opportunities in real code. It can be // removed at a later date. if (pcRelInsn == UINT64_C(-1)) { errorOrWarn( "unrecognized instruction for R_PPC64_PCREL_OPT relaxation: 0x" + Twine::utohexstr(accessInsn)); break; } int64_t totalDisp = getTotalDisp(insn, accessInsn); if (!isInt<34>(totalDisp)) break; // Displacement doesn't fit. // Convert the PADDI to the prefixed version of accessInsn and convert // accessInsn to a nop. writePrefixedInstruction(loc, pcRelInsn | ((totalDisp & 0x3ffff0000) << 16) | (totalDisp & 0xffff)); write32(loc + rel.addend, NOP); // nop accessInsn. break; } default: llvm_unreachable("unexpected relocation type"); } } void PPC64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const { // Reference: 3.7.4.2 of the 64-bit ELF V2 abi supplement. // The general dynamic code sequence for a global `x` will look like: // Instruction Relocation Symbol // addis r3, r2, x@got@tlsgd@ha R_PPC64_GOT_TLSGD16_HA x // addi r3, r3, x@got@tlsgd@l R_PPC64_GOT_TLSGD16_LO x // bl __tls_get_addr(x@tlsgd) R_PPC64_TLSGD x // R_PPC64_REL24 __tls_get_addr // nop None None // Relaxing to local exec entails converting: // addis r3, r2, x@got@tlsgd@ha into nop // addi r3, r3, x@got@tlsgd@l into addis r3, r13, x@tprel@ha // bl __tls_get_addr(x@tlsgd) into nop // nop into addi r3, r3, x@tprel@l switch (rel.type) { case R_PPC64_GOT_TLSGD16_HA: writeFromHalf16(loc, NOP); break; case R_PPC64_GOT_TLSGD16: case R_PPC64_GOT_TLSGD16_LO: writeFromHalf16(loc, 0x3c6d0000); // addis r3, r13 relocateNoSym(loc, R_PPC64_TPREL16_HA, val); break; - case R_PPC64_GOT_TLSGD_PCREL34: - // Relax from paddi r3, 0, x@got@tlsgd@pcrel, 1 to - // paddi r3, r13, x@tprel, 0 - writePrefixedInstruction(loc, 0x06000000386d0000); - relocateNoSym(loc, R_PPC64_TPREL34, val); - break; - case R_PPC64_TLSGD: { - // PC Relative Relaxation: - // Relax from bl __tls_get_addr@notoc(x@tlsgd) to - // nop - // TOC Relaxation: - // Relax from bl __tls_get_addr(x@tlsgd) - // nop - // to - // nop - // addi r3, r3, x@tprel@l - const uintptr_t locAsInt = reinterpret_cast(loc); - if (locAsInt % 4 == 0) { - write32(loc, NOP); // nop - write32(loc + 4, 0x38630000); // addi r3, r3 - // Since we are relocating a half16 type relocation and Loc + 4 points to - // the start of an instruction we need to advance the buffer by an extra - // 2 bytes on BE. - relocateNoSym(loc + 4 + (config->ekind == ELF64BEKind ? 2 : 0), - R_PPC64_TPREL16_LO, val); - } else if (locAsInt % 4 == 1) { - write32(loc - 1, NOP); - } else { - errorOrWarn("R_PPC64_TLSGD has unexpected byte alignment"); - } + case R_PPC64_TLSGD: + write32(loc, NOP); + write32(loc + 4, 0x38630000); // addi r3, r3 + // Since we are relocating a half16 type relocation and Loc + 4 points to + // the start of an instruction we need to advance the buffer by an extra + // 2 bytes on BE. + relocateNoSym(loc + 4 + (config->ekind == ELF64BEKind ? 2 : 0), + R_PPC64_TPREL16_LO, val); break; - } default: llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); } } void PPC64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const { // Reference: 3.7.4.3 of the 64-bit ELF V2 abi supplement. // The local dynamic code sequence for a global `x` will look like: // Instruction Relocation Symbol // addis r3, r2, x@got@tlsld@ha R_PPC64_GOT_TLSLD16_HA x // addi r3, r3, x@got@tlsld@l R_PPC64_GOT_TLSLD16_LO x // bl __tls_get_addr(x@tlsgd) R_PPC64_TLSLD x // R_PPC64_REL24 __tls_get_addr // nop None None // Relaxing to local exec entails converting: // addis r3, r2, x@got@tlsld@ha into nop // addi r3, r3, x@got@tlsld@l into addis r3, r13, 0 // bl __tls_get_addr(x@tlsgd) into nop // nop into addi r3, r3, 4096 switch (rel.type) { case R_PPC64_GOT_TLSLD16_HA: writeFromHalf16(loc, NOP); break; case R_PPC64_GOT_TLSLD16_LO: writeFromHalf16(loc, 0x3c6d0000); // addis r3, r13, 0 break; case R_PPC64_TLSLD: write32(loc, NOP); write32(loc + 4, 0x38631000); // addi r3, r3, 4096 break; case R_PPC64_DTPREL16: case R_PPC64_DTPREL16_HA: case R_PPC64_DTPREL16_HI: case R_PPC64_DTPREL16_DS: case R_PPC64_DTPREL16_LO: case R_PPC64_DTPREL16_LO_DS: relocate(loc, rel, val); break; default: llvm_unreachable("unsupported relocation for TLS LD to LE relaxation"); } } unsigned elf::getPPCDFormOp(unsigned secondaryOp) { switch (secondaryOp) { case LBZX: return LBZ; case LHZX: return LHZ; case LWZX: return LWZ; case LDX: return LD; case STBX: return STB; case STHX: return STH; case STWX: return STW; case STDX: return STD; case ADD: return ADDI; default: return 0; } } void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const { // The initial exec code sequence for a global `x` will look like: // Instruction Relocation Symbol // addis r9, r2, x@got@tprel@ha R_PPC64_GOT_TPREL16_HA x // ld r9, x@got@tprel@l(r9) R_PPC64_GOT_TPREL16_LO_DS x // add r9, r9, x@tls R_PPC64_TLS x // Relaxing to local exec entails converting: // addis r9, r2, x@got@tprel@ha into nop // ld r9, x@got@tprel@l(r9) into addis r9, r13, x@tprel@ha // add r9, r9, x@tls into addi r9, r9, x@tprel@l // x@tls R_PPC64_TLS is a relocation which does not compute anything, // it is replaced with r13 (thread pointer). // The add instruction in the initial exec sequence has multiple variations // that need to be handled. If we are building an address it will use an add // instruction, if we are accessing memory it will use any of the X-form // indexed load or store instructions. unsigned offset = (config->ekind == ELF64BEKind) ? 2 : 0; switch (rel.type) { case R_PPC64_GOT_TPREL16_HA: write32(loc - offset, NOP); break; case R_PPC64_GOT_TPREL16_LO_DS: case R_PPC64_GOT_TPREL16_DS: { uint32_t regNo = read32(loc - offset) & 0x03E00000; // bits 6-10 write32(loc - offset, 0x3C0D0000 | regNo); // addis RegNo, r13 relocateNoSym(loc, R_PPC64_TPREL16_HA, val); break; } case R_PPC64_GOT_TPREL_PCREL34: { const uint64_t pldRT = readPrefixedInstruction(loc) & 0x0000000003e00000; // paddi RT(from pld), r13, symbol@tprel, 0 writePrefixedInstruction(loc, 0x06000000380d0000 | pldRT); relocateNoSym(loc, R_PPC64_TPREL34, val); break; } case R_PPC64_TLS: { const uintptr_t locAsInt = reinterpret_cast(loc); if (locAsInt % 4 == 0) { uint32_t primaryOp = getPrimaryOpCode(read32(loc)); if (primaryOp != 31) error("unrecognized instruction for IE to LE R_PPC64_TLS"); uint32_t secondaryOp = (read32(loc) & 0x000007FE) >> 1; // bits 21-30 uint32_t dFormOp = getPPCDFormOp(secondaryOp); if (dFormOp == 0) error("unrecognized instruction for IE to LE R_PPC64_TLS"); write32(loc, ((dFormOp << 26) | (read32(loc) & 0x03FFFFFF))); relocateNoSym(loc + offset, R_PPC64_TPREL16_LO, val); } else if (locAsInt % 4 == 1) { // If the offset is not 4 byte aligned then we have a PCRel type reloc. // This version of the relocation is offset by one byte from the // instruction it references. uint32_t tlsInstr = read32(loc - 1); uint32_t primaryOp = getPrimaryOpCode(tlsInstr); if (primaryOp != 31) errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS"); uint32_t secondaryOp = (tlsInstr & 0x000007FE) >> 1; // bits 21-30 // The add is a special case and should be turned into a nop. The paddi // that comes before it will already have computed the address of the // symbol. if (secondaryOp == 266) { write32(loc - 1, NOP); } else { uint32_t dFormOp = getPPCDFormOp(secondaryOp); if (dFormOp == 0) errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS"); write32(loc - 1, ((dFormOp << 26) | (tlsInstr & 0x03FF0000))); } } else { errorOrWarn("R_PPC64_TLS must be either 4 byte aligned or one byte " "offset from 4 byte aligned"); } break; } default: llvm_unreachable("unknown relocation for IE to LE"); break; } } RelExpr PPC64::getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const { switch (type) { case R_PPC64_NONE: return R_NONE; case R_PPC64_ADDR16: case R_PPC64_ADDR16_DS: case R_PPC64_ADDR16_HA: case R_PPC64_ADDR16_HI: case R_PPC64_ADDR16_HIGHER: case R_PPC64_ADDR16_HIGHERA: case R_PPC64_ADDR16_HIGHEST: case R_PPC64_ADDR16_HIGHESTA: case R_PPC64_ADDR16_LO: case R_PPC64_ADDR16_LO_DS: case R_PPC64_ADDR32: case R_PPC64_ADDR64: return R_ABS; case R_PPC64_GOT16: case R_PPC64_GOT16_DS: case R_PPC64_GOT16_HA: case R_PPC64_GOT16_HI: case R_PPC64_GOT16_LO: case R_PPC64_GOT16_LO_DS: return R_GOT_OFF; case R_PPC64_TOC16: case R_PPC64_TOC16_DS: case R_PPC64_TOC16_HI: case R_PPC64_TOC16_LO: return R_GOTREL; case R_PPC64_GOT_PCREL34: case R_PPC64_GOT_TPREL_PCREL34: case R_PPC64_PCREL_OPT: return R_GOT_PC; case R_PPC64_TOC16_HA: case R_PPC64_TOC16_LO_DS: return config->tocOptimize ? R_PPC64_RELAX_TOC : R_GOTREL; case R_PPC64_TOC: return R_PPC64_TOCBASE; case R_PPC64_REL14: case R_PPC64_REL24: return R_PPC64_CALL_PLT; case R_PPC64_REL24_NOTOC: return R_PLT_PC; case R_PPC64_REL16_LO: case R_PPC64_REL16_HA: case R_PPC64_REL16_HI: case R_PPC64_REL32: case R_PPC64_REL64: case R_PPC64_PCREL34: return R_PC; case R_PPC64_GOT_TLSGD16: case R_PPC64_GOT_TLSGD16_HA: case R_PPC64_GOT_TLSGD16_HI: case R_PPC64_GOT_TLSGD16_LO: return R_TLSGD_GOT; - case R_PPC64_GOT_TLSGD_PCREL34: - return R_TLSGD_PC; case R_PPC64_GOT_TLSLD16: case R_PPC64_GOT_TLSLD16_HA: case R_PPC64_GOT_TLSLD16_HI: case R_PPC64_GOT_TLSLD16_LO: return R_TLSLD_GOT; case R_PPC64_GOT_TPREL16_HA: case R_PPC64_GOT_TPREL16_LO_DS: case R_PPC64_GOT_TPREL16_DS: case R_PPC64_GOT_TPREL16_HI: return R_GOT_OFF; case R_PPC64_GOT_DTPREL16_HA: case R_PPC64_GOT_DTPREL16_LO_DS: case R_PPC64_GOT_DTPREL16_DS: case R_PPC64_GOT_DTPREL16_HI: return R_TLSLD_GOT_OFF; case R_PPC64_TPREL16: case R_PPC64_TPREL16_HA: case R_PPC64_TPREL16_LO: case R_PPC64_TPREL16_HI: case R_PPC64_TPREL16_DS: case R_PPC64_TPREL16_LO_DS: case R_PPC64_TPREL16_HIGHER: case R_PPC64_TPREL16_HIGHERA: case R_PPC64_TPREL16_HIGHEST: case R_PPC64_TPREL16_HIGHESTA: case R_PPC64_TPREL34: return R_TLS; case R_PPC64_DTPREL16: case R_PPC64_DTPREL16_DS: case R_PPC64_DTPREL16_HA: case R_PPC64_DTPREL16_HI: case R_PPC64_DTPREL16_HIGHER: case R_PPC64_DTPREL16_HIGHERA: case R_PPC64_DTPREL16_HIGHEST: case R_PPC64_DTPREL16_HIGHESTA: case R_PPC64_DTPREL16_LO: case R_PPC64_DTPREL16_LO_DS: case R_PPC64_DTPREL64: return R_DTPREL; case R_PPC64_TLSGD: return R_TLSDESC_CALL; case R_PPC64_TLSLD: return R_TLSLD_HINT; case R_PPC64_TLS: return R_TLSIE_HINT; default: error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + ") against symbol " + toString(s)); return R_NONE; } } RelType PPC64::getDynRel(RelType type) const { if (type == R_PPC64_ADDR64 || type == R_PPC64_TOC) return R_PPC64_ADDR64; return R_PPC64_NONE; } void PPC64::writeGotHeader(uint8_t *buf) const { write64(buf, getPPC64TocBase()); } void PPC64::writePltHeader(uint8_t *buf) const { // The generic resolver stub goes first. write32(buf + 0, 0x7c0802a6); // mflr r0 write32(buf + 4, 0x429f0005); // bcl 20,4*cr7+so,8 <_glink+0x8> write32(buf + 8, 0x7d6802a6); // mflr r11 write32(buf + 12, 0x7c0803a6); // mtlr r0 write32(buf + 16, 0x7d8b6050); // subf r12, r11, r12 write32(buf + 20, 0x380cffcc); // subi r0,r12,52 write32(buf + 24, 0x7800f082); // srdi r0,r0,62,2 write32(buf + 28, 0xe98b002c); // ld r12,44(r11) write32(buf + 32, 0x7d6c5a14); // add r11,r12,r11 write32(buf + 36, 0xe98b0000); // ld r12,0(r11) write32(buf + 40, 0xe96b0008); // ld r11,8(r11) write32(buf + 44, 0x7d8903a6); // mtctr r12 write32(buf + 48, 0x4e800420); // bctr // The 'bcl' instruction will set the link register to the address of the // following instruction ('mflr r11'). Here we store the offset from that // instruction to the first entry in the GotPlt section. int64_t gotPltOffset = in.gotPlt->getVA() - (in.plt->getVA() + 8); write64(buf + 52, gotPltOffset); } void PPC64::writePlt(uint8_t *buf, const Symbol &sym, uint64_t /*pltEntryAddr*/) const { int32_t offset = pltHeaderSize + sym.pltIndex * pltEntrySize; // bl __glink_PLTresolve write32(buf, 0x48000000 | ((-offset) & 0x03FFFFFc)); } void PPC64::writeIplt(uint8_t *buf, const Symbol &sym, uint64_t /*pltEntryAddr*/) const { writePPC64LoadAndBranch(buf, sym.getGotPltVA() - getPPC64TocBase()); } static std::pair toAddr16Rel(RelType type, uint64_t val) { // Relocations relative to the toc-base need to be adjusted by the Toc offset. uint64_t tocBiasedVal = val - ppc64TocOffset; // Relocations relative to dtv[dtpmod] need to be adjusted by the DTP offset. uint64_t dtpBiasedVal = val - dynamicThreadPointerOffset; switch (type) { // TOC biased relocation. case R_PPC64_GOT16: case R_PPC64_GOT_TLSGD16: case R_PPC64_GOT_TLSLD16: case R_PPC64_TOC16: return {R_PPC64_ADDR16, tocBiasedVal}; case R_PPC64_GOT16_DS: case R_PPC64_TOC16_DS: case R_PPC64_GOT_TPREL16_DS: case R_PPC64_GOT_DTPREL16_DS: return {R_PPC64_ADDR16_DS, tocBiasedVal}; case R_PPC64_GOT16_HA: case R_PPC64_GOT_TLSGD16_HA: case R_PPC64_GOT_TLSLD16_HA: case R_PPC64_GOT_TPREL16_HA: case R_PPC64_GOT_DTPREL16_HA: case R_PPC64_TOC16_HA: return {R_PPC64_ADDR16_HA, tocBiasedVal}; case R_PPC64_GOT16_HI: case R_PPC64_GOT_TLSGD16_HI: case R_PPC64_GOT_TLSLD16_HI: case R_PPC64_GOT_TPREL16_HI: case R_PPC64_GOT_DTPREL16_HI: case R_PPC64_TOC16_HI: return {R_PPC64_ADDR16_HI, tocBiasedVal}; case R_PPC64_GOT16_LO: case R_PPC64_GOT_TLSGD16_LO: case R_PPC64_GOT_TLSLD16_LO: case R_PPC64_TOC16_LO: return {R_PPC64_ADDR16_LO, tocBiasedVal}; case R_PPC64_GOT16_LO_DS: case R_PPC64_TOC16_LO_DS: case R_PPC64_GOT_TPREL16_LO_DS: case R_PPC64_GOT_DTPREL16_LO_DS: return {R_PPC64_ADDR16_LO_DS, tocBiasedVal}; // Dynamic Thread pointer biased relocation types. case R_PPC64_DTPREL16: return {R_PPC64_ADDR16, dtpBiasedVal}; case R_PPC64_DTPREL16_DS: return {R_PPC64_ADDR16_DS, dtpBiasedVal}; case R_PPC64_DTPREL16_HA: return {R_PPC64_ADDR16_HA, dtpBiasedVal}; case R_PPC64_DTPREL16_HI: return {R_PPC64_ADDR16_HI, dtpBiasedVal}; case R_PPC64_DTPREL16_HIGHER: return {R_PPC64_ADDR16_HIGHER, dtpBiasedVal}; case R_PPC64_DTPREL16_HIGHERA: return {R_PPC64_ADDR16_HIGHERA, dtpBiasedVal}; case R_PPC64_DTPREL16_HIGHEST: return {R_PPC64_ADDR16_HIGHEST, dtpBiasedVal}; case R_PPC64_DTPREL16_HIGHESTA: return {R_PPC64_ADDR16_HIGHESTA, dtpBiasedVal}; case R_PPC64_DTPREL16_LO: return {R_PPC64_ADDR16_LO, dtpBiasedVal}; case R_PPC64_DTPREL16_LO_DS: return {R_PPC64_ADDR16_LO_DS, dtpBiasedVal}; case R_PPC64_DTPREL64: return {R_PPC64_ADDR64, dtpBiasedVal}; default: return {type, val}; } } static bool isTocOptType(RelType type) { switch (type) { case R_PPC64_GOT16_HA: case R_PPC64_GOT16_LO_DS: case R_PPC64_TOC16_HA: case R_PPC64_TOC16_LO_DS: case R_PPC64_TOC16_LO: return true; default: return false; } } void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { RelType type = rel.type; bool shouldTocOptimize = isTocOptType(type); // For dynamic thread pointer relative, toc-relative, and got-indirect // relocations, proceed in terms of the corresponding ADDR16 relocation type. std::tie(type, val) = toAddr16Rel(type, val); switch (type) { case R_PPC64_ADDR14: { checkAlignment(loc, val, 4, rel); // Preserve the AA/LK bits in the branch instruction uint8_t aalk = loc[3]; write16(loc + 2, (aalk & 3) | (val & 0xfffc)); break; } case R_PPC64_ADDR16: checkIntUInt(loc, val, 16, rel); write16(loc, val); break; case R_PPC64_ADDR32: checkIntUInt(loc, val, 32, rel); write32(loc, val); break; case R_PPC64_ADDR16_DS: case R_PPC64_TPREL16_DS: { checkInt(loc, val, 16, rel); // DQ-form instructions use bits 28-31 as part of the instruction encoding // DS-form instructions only use bits 30-31. uint16_t mask = isDQFormInstruction(readFromHalf16(loc)) ? 0xf : 0x3; checkAlignment(loc, lo(val), mask + 1, rel); write16(loc, (read16(loc) & mask) | lo(val)); } break; case R_PPC64_ADDR16_HA: case R_PPC64_REL16_HA: case R_PPC64_TPREL16_HA: if (config->tocOptimize && shouldTocOptimize && ha(val) == 0) writeFromHalf16(loc, NOP); else write16(loc, ha(val)); break; case R_PPC64_ADDR16_HI: case R_PPC64_REL16_HI: case R_PPC64_TPREL16_HI: write16(loc, hi(val)); break; case R_PPC64_ADDR16_HIGHER: case R_PPC64_TPREL16_HIGHER: write16(loc, higher(val)); break; case R_PPC64_ADDR16_HIGHERA: case R_PPC64_TPREL16_HIGHERA: write16(loc, highera(val)); break; case R_PPC64_ADDR16_HIGHEST: case R_PPC64_TPREL16_HIGHEST: write16(loc, highest(val)); break; case R_PPC64_ADDR16_HIGHESTA: case R_PPC64_TPREL16_HIGHESTA: write16(loc, highesta(val)); break; case R_PPC64_ADDR16_LO: case R_PPC64_REL16_LO: case R_PPC64_TPREL16_LO: // When the high-adjusted part of a toc relocation evaluates to 0, it is // changed into a nop. The lo part then needs to be updated to use the // toc-pointer register r2, as the base register. if (config->tocOptimize && shouldTocOptimize && ha(val) == 0) { uint32_t insn = readFromHalf16(loc); if (isInstructionUpdateForm(insn)) error(getErrorLocation(loc) + "can't toc-optimize an update instruction: 0x" + utohexstr(insn)); writeFromHalf16(loc, (insn & 0xffe00000) | 0x00020000 | lo(val)); } else { write16(loc, lo(val)); } break; case R_PPC64_ADDR16_LO_DS: case R_PPC64_TPREL16_LO_DS: { // DQ-form instructions use bits 28-31 as part of the instruction encoding // DS-form instructions only use bits 30-31. uint32_t insn = readFromHalf16(loc); uint16_t mask = isDQFormInstruction(insn) ? 0xf : 0x3; checkAlignment(loc, lo(val), mask + 1, rel); if (config->tocOptimize && shouldTocOptimize && ha(val) == 0) { // When the high-adjusted part of a toc relocation evaluates to 0, it is // changed into a nop. The lo part then needs to be updated to use the toc // pointer register r2, as the base register. if (isInstructionUpdateForm(insn)) error(getErrorLocation(loc) + "Can't toc-optimize an update instruction: 0x" + Twine::utohexstr(insn)); insn &= 0xffe00000 | mask; writeFromHalf16(loc, insn | 0x00020000 | lo(val)); } else { write16(loc, (read16(loc) & mask) | lo(val)); } } break; case R_PPC64_TPREL16: checkInt(loc, val, 16, rel); write16(loc, val); break; case R_PPC64_REL32: checkInt(loc, val, 32, rel); write32(loc, val); break; case R_PPC64_ADDR64: case R_PPC64_REL64: case R_PPC64_TOC: write64(loc, val); break; case R_PPC64_REL14: { uint32_t mask = 0x0000FFFC; checkInt(loc, val, 16, rel); checkAlignment(loc, val, 4, rel); write32(loc, (read32(loc) & ~mask) | (val & mask)); break; } case R_PPC64_REL24: case R_PPC64_REL24_NOTOC: { uint32_t mask = 0x03FFFFFC; checkInt(loc, val, 26, rel); checkAlignment(loc, val, 4, rel); write32(loc, (read32(loc) & ~mask) | (val & mask)); break; } case R_PPC64_DTPREL64: write64(loc, val - dynamicThreadPointerOffset); break; case R_PPC64_PCREL34: case R_PPC64_GOT_PCREL34: - case R_PPC64_GOT_TLSGD_PCREL34: case R_PPC64_GOT_TPREL_PCREL34: case R_PPC64_TPREL34: { const uint64_t si0Mask = 0x00000003ffff0000; const uint64_t si1Mask = 0x000000000000ffff; const uint64_t fullMask = 0x0003ffff0000ffff; checkInt(loc, val, 34, rel); uint64_t instr = readPrefixedInstruction(loc) & ~fullMask; writePrefixedInstruction(loc, instr | ((val & si0Mask) << 16) | (val & si1Mask)); break; } // If we encounter a PCREL_OPT relocation that we won't optimize. case R_PPC64_PCREL_OPT: break; default: llvm_unreachable("unknown relocation"); } } bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file, uint64_t branchAddr, const Symbol &s, int64_t a) const { if (type != R_PPC64_REL14 && type != R_PPC64_REL24 && type != R_PPC64_REL24_NOTOC) return false; // If a function is in the Plt it needs to be called with a call-stub. if (s.isInPlt()) return true; // This check looks at the st_other bits of the callee with relocation // R_PPC64_REL14 or R_PPC64_REL24. If the value is 1, then the callee // clobbers the TOC and we need an R2 save stub. if (type != R_PPC64_REL24_NOTOC && (s.stOther >> 5) == 1) return true; if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1) return true; // If a symbol is a weak undefined and we are compiling an executable // it doesn't need a range-extending thunk since it can't be called. if (s.isUndefWeak() && !config->shared) return false; // If the offset exceeds the range of the branch type then it will need // a range-extending thunk. // See the comment in getRelocTargetVA() about R_PPC64_CALL. return !inBranchRange(type, branchAddr, s.getVA(a) + getPPC64GlobalEntryToLocalEntryOffset(s.stOther)); } uint32_t PPC64::getThunkSectionSpacing() const { // See comment in Arch/ARM.cpp for a more detailed explanation of // getThunkSectionSpacing(). For PPC64 we pick the constant here based on // R_PPC64_REL24, which is used by unconditional branch instructions. // 0x2000000 = (1 << 24-1) * 4 return 0x2000000; } bool PPC64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { int64_t offset = dst - src; if (type == R_PPC64_REL14) return isInt<16>(offset); if (type == R_PPC64_REL24 || type == R_PPC64_REL24_NOTOC) return isInt<26>(offset); llvm_unreachable("unsupported relocation type used in branch"); } RelExpr PPC64::adjustRelaxExpr(RelType type, const uint8_t *data, RelExpr expr) const { if ((type == R_PPC64_GOT_PCREL34 || type == R_PPC64_PCREL_OPT) && config->pcRelOptimize) { // It only makes sense to optimize pld since paddi means that the address // of the object in the GOT is required rather than the object itself. assert(data && "Expecting an instruction encoding here"); if ((readPrefixedInstruction(data) & 0xfc000000) == 0xe4000000) return R_PPC64_RELAX_GOT_PC; } - - if (type != R_PPC64_GOT_TLSGD_PCREL34 && expr == R_RELAX_TLS_GD_TO_IE) + if (expr == R_RELAX_TLS_GD_TO_IE) return R_RELAX_TLS_GD_TO_IE_GOT_OFF; if (expr == R_RELAX_TLS_LD_TO_LE) return R_RELAX_TLS_LD_TO_LE_ABS; return expr; } // Reference: 3.7.4.1 of the 64-bit ELF V2 abi supplement. // The general dynamic code sequence for a global `x` uses 4 instructions. // Instruction Relocation Symbol // addis r3, r2, x@got@tlsgd@ha R_PPC64_GOT_TLSGD16_HA x // addi r3, r3, x@got@tlsgd@l R_PPC64_GOT_TLSGD16_LO x // bl __tls_get_addr(x@tlsgd) R_PPC64_TLSGD x // R_PPC64_REL24 __tls_get_addr // nop None None // // Relaxing to initial-exec entails: // 1) Convert the addis/addi pair that builds the address of the tls_index // struct for 'x' to an addis/ld pair that loads an offset from a got-entry. // 2) Convert the call to __tls_get_addr to a nop. // 3) Convert the nop following the call to an add of the loaded offset to the // thread pointer. // Since the nop must directly follow the call, the R_PPC64_TLSGD relocation is // used as the relaxation hint for both steps 2 and 3. void PPC64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const { switch (rel.type) { case R_PPC64_GOT_TLSGD16_HA: // This is relaxed from addis rT, r2, sym@got@tlsgd@ha to // addis rT, r2, sym@got@tprel@ha. relocateNoSym(loc, R_PPC64_GOT_TPREL16_HA, val); return; case R_PPC64_GOT_TLSGD16: case R_PPC64_GOT_TLSGD16_LO: { // Relax from addi r3, rA, sym@got@tlsgd@l to // ld r3, sym@got@tprel@l(rA) uint32_t ra = (readFromHalf16(loc) & (0x1f << 16)); writeFromHalf16(loc, 0xe8600000 | ra); relocateNoSym(loc, R_PPC64_GOT_TPREL16_LO_DS, val); return; } - case R_PPC64_GOT_TLSGD_PCREL34: { - // Relax from paddi r3, 0, sym@got@tlsgd@pcrel, 1 to - // pld r3, sym@got@tprel@pcrel - writePrefixedInstruction(loc, 0x04100000e4600000); - relocateNoSym(loc, R_PPC64_GOT_TPREL_PCREL34, val); - return; - } - case R_PPC64_TLSGD: { - // PC Relative Relaxation: - // Relax from bl __tls_get_addr@notoc(x@tlsgd) to - // nop - // TOC Relaxation: - // Relax from bl __tls_get_addr(x@tlsgd) - // nop - // to - // nop - // add r3, r3, r13 - const uintptr_t locAsInt = reinterpret_cast(loc); - if (locAsInt % 4 == 0) { - write32(loc, NOP); // bl __tls_get_addr(sym@tlsgd) --> nop - write32(loc + 4, 0x7c636A14); // nop --> add r3, r3, r13 - } else if (locAsInt % 4 == 1) { - // bl __tls_get_addr(sym@tlsgd) --> add r3, r3, r13 - write32(loc - 1, 0x7c636a14); - } else { - errorOrWarn("R_PPC64_TLSGD has unexpected byte alignment"); - } + case R_PPC64_TLSGD: + write32(loc, NOP); // bl __tls_get_addr(sym@tlsgd) --> nop + write32(loc + 4, 0x7c636A14); // nop --> add r3, r3, r13 return; - } default: llvm_unreachable("unsupported relocation for TLS GD to IE relaxation"); } } // The prologue for a split-stack function is expected to look roughly // like this: // .Lglobal_entry_point: // # TOC pointer initialization. // ... // .Llocal_entry_point: // # load the __private_ss member of the threads tcbhead. // ld r0,-0x7000-64(r13) // # subtract the functions stack size from the stack pointer. // addis r12, r1, ha(-stack-frame size) // addi r12, r12, l(-stack-frame size) // # compare needed to actual and branch to allocate_more_stack if more // # space is needed, otherwise fallthrough to 'normal' function body. // cmpld cr7,r12,r0 // blt- cr7, .Lallocate_more_stack // // -) The allocate_more_stack block might be placed after the split-stack // prologue and the `blt-` replaced with a `bge+ .Lnormal_func_body` // instead. // -) If either the addis or addi is not needed due to the stack size being // smaller then 32K or a multiple of 64K they will be replaced with a nop, // but there will always be 2 instructions the linker can overwrite for the // adjusted stack size. // // The linkers job here is to increase the stack size used in the addis/addi // pair by split-stack-size-adjust. // addis r12, r1, ha(-stack-frame size - split-stack-adjust-size) // addi r12, r12, l(-stack-frame size - split-stack-adjust-size) bool PPC64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, uint8_t stOther) const { // If the caller has a global entry point adjust the buffer past it. The start // of the split-stack prologue will be at the local entry point. loc += getPPC64GlobalEntryToLocalEntryOffset(stOther); // At the very least we expect to see a load of some split-stack data from the // tcb, and 2 instructions that calculate the ending stack address this // function will require. If there is not enough room for at least 3 // instructions it can't be a split-stack prologue. if (loc + 12 >= end) return false; // First instruction must be `ld r0, -0x7000-64(r13)` if (read32(loc) != 0xe80d8fc0) return false; int16_t hiImm = 0; int16_t loImm = 0; // First instruction can be either an addis if the frame size is larger then // 32K, or an addi if the size is less then 32K. int32_t firstInstr = read32(loc + 4); if (getPrimaryOpCode(firstInstr) == 15) { hiImm = firstInstr & 0xFFFF; } else if (getPrimaryOpCode(firstInstr) == 14) { loImm = firstInstr & 0xFFFF; } else { return false; } // Second instruction is either an addi or a nop. If the first instruction was // an addi then LoImm is set and the second instruction must be a nop. uint32_t secondInstr = read32(loc + 8); if (!loImm && getPrimaryOpCode(secondInstr) == 14) { loImm = secondInstr & 0xFFFF; } else if (secondInstr != NOP) { return false; } // The register operands of the first instruction should be the stack-pointer // (r1) as the input (RA) and r12 as the output (RT). If the second // instruction is not a nop, then it should use r12 as both input and output. auto checkRegOperands = [](uint32_t instr, uint8_t expectedRT, uint8_t expectedRA) { return ((instr & 0x3E00000) >> 21 == expectedRT) && ((instr & 0x1F0000) >> 16 == expectedRA); }; if (!checkRegOperands(firstInstr, 12, 1)) return false; if (secondInstr != NOP && !checkRegOperands(secondInstr, 12, 12)) return false; int32_t stackFrameSize = (hiImm * 65536) + loImm; // Check that the adjusted size doesn't overflow what we can represent with 2 // instructions. if (stackFrameSize < config->splitStackAdjustSize + INT32_MIN) { error(getErrorLocation(loc) + "split-stack prologue adjustment overflows"); return false; } int32_t adjustedStackFrameSize = stackFrameSize - config->splitStackAdjustSize; loImm = adjustedStackFrameSize & 0xFFFF; hiImm = (adjustedStackFrameSize + 0x8000) >> 16; if (hiImm) { write32(loc + 4, 0x3D810000 | (uint16_t)hiImm); // If the low immediate is zero the second instruction will be a nop. secondInstr = loImm ? 0x398C0000 | (uint16_t)loImm : NOP; write32(loc + 8, secondInstr); } else { // addi r12, r1, imm write32(loc + 4, (0x39810000) | (uint16_t)loImm); write32(loc + 8, NOP); } return true; } TargetInfo *elf::getPPC64TargetInfo() { static PPC64 target; return ⌖ } diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index ea6aa3c6a12a..4c6a70d9034e 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1,2089 +1,2076 @@ //===- Relocations.cpp ----------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains platform-independent functions to process relocations. // I'll describe the overview of this file here. // // Simple relocations are easy to handle for the linker. For example, // for R_X86_64_PC64 relocs, the linker just has to fix up locations // with the relative offsets to the target symbols. It would just be // reading records from relocation sections and applying them to output. // // But not all relocations are that easy to handle. For example, for // R_386_GOTOFF relocs, the linker has to create new GOT entries for // symbols if they don't exist, and fix up locations with GOT entry // offsets from the beginning of GOT section. So there is more than // fixing addresses in relocation processing. // // ELF defines a large number of complex relocations. // // The functions in this file analyze relocations and do whatever needs // to be done. It includes, but not limited to, the following. // // - create GOT/PLT entries // - create new relocations in .dynsym to let the dynamic linker resolve // them at runtime (since ELF supports dynamic linking, not all // relocations can be resolved at link-time) // - create COPY relocs and reserve space in .bss // - replace expensive relocs (in terms of runtime cost) with cheap ones // - error out infeasible combinations such as PIC and non-relative relocs // // Note that the functions in this file don't actually apply relocations // because it doesn't know about the output file nor the output file buffer. // It instead stores Relocation objects to InputSection's Relocations // vector to let it apply later in InputSection::writeTo. // //===----------------------------------------------------------------------===// #include "Relocations.h" #include "Config.h" #include "LinkerScript.h" #include "OutputSections.h" #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" #include "Thunks.h" #include "lld/Common/ErrorHandler.h" #include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Demangle/Demangle.h" #include "llvm/Support/Endian.h" #include "llvm/Support/raw_ostream.h" #include using namespace llvm; using namespace llvm::ELF; using namespace llvm::object; using namespace llvm::support::endian; using namespace lld; using namespace lld::elf; static Optional getLinkerScriptLocation(const Symbol &sym) { for (BaseCommand *base : script->sectionCommands) if (auto *cmd = dyn_cast(base)) if (cmd->sym == &sym) return cmd->location; return None; } static std::string getDefinedLocation(const Symbol &sym) { std::string msg = "\n>>> defined in "; if (sym.file) msg += toString(sym.file); else if (Optional loc = getLinkerScriptLocation(sym)) msg += *loc; return msg; } // Construct a message in the following format. // // >>> defined in /home/alice/src/foo.o // >>> referenced by bar.c:12 (/home/alice/src/bar.c:12) // >>> /home/alice/src/bar.o:(.text+0x1) static std::string getLocation(InputSectionBase &s, const Symbol &sym, uint64_t off) { std::string msg = getDefinedLocation(sym) + "\n>>> referenced by "; std::string src = s.getSrcMsg(sym, off); if (!src.empty()) msg += src + "\n>>> "; return msg + s.getObjMsg(off); } void elf::reportRangeError(uint8_t *loc, const Relocation &rel, const Twine &v, int64_t min, uint64_t max) { ErrorPlace errPlace = getErrorPlace(loc); std::string hint; if (rel.sym && !rel.sym->isLocal()) hint = "; references " + lld::toString(*rel.sym) + getDefinedLocation(*rel.sym); if (errPlace.isec && errPlace.isec->name.startswith(".debug")) hint += "; consider recompiling with -fdebug-types-section to reduce size " "of debug sections"; errorOrWarn(errPlace.loc + "relocation " + lld::toString(rel.type) + " out of range: " + v.str() + " is not in [" + Twine(min).str() + ", " + Twine(max).str() + "]" + hint); } void elf::reportRangeError(uint8_t *loc, int64_t v, int n, const Symbol &sym, const Twine &msg) { ErrorPlace errPlace = getErrorPlace(loc); std::string hint; if (!sym.getName().empty()) hint = "; references " + lld::toString(sym) + getDefinedLocation(sym); errorOrWarn(errPlace.loc + msg + " is out of range: " + Twine(v) + " is not in [" + Twine(llvm::minIntN(n)) + ", " + Twine(llvm::maxIntN(n)) + "]" + hint); } namespace { // Build a bitmask with one bit set for each RelExpr. // // Constexpr function arguments can't be used in static asserts, so we // use template arguments to build the mask. // But function template partial specializations don't exist (needed // for base case of the recursion), so we need a dummy struct. template struct RelExprMaskBuilder { static inline uint64_t build() { return 0; } }; // Specialization for recursive case. template struct RelExprMaskBuilder { static inline uint64_t build() { static_assert(0 <= Head && Head < 64, "RelExpr is too large for 64-bit mask!"); return (uint64_t(1) << Head) | RelExprMaskBuilder::build(); } }; } // namespace // Return true if `Expr` is one of `Exprs`. // There are fewer than 64 RelExpr's, so we can represent any set of // RelExpr's as a constant bit mask and test for membership with a // couple cheap bitwise operations. template bool oneof(RelExpr expr) { assert(0 <= expr && (int)expr < 64 && "RelExpr is too large for 64-bit mask!"); return (uint64_t(1) << expr) & RelExprMaskBuilder::build(); } // This function is similar to the `handleTlsRelocation`. MIPS does not // support any relaxations for TLS relocations so by factoring out MIPS // handling in to the separate function we can simplify the code and do not // pollute other `handleTlsRelocation` by MIPS `ifs` statements. // Mips has a custom MipsGotSection that handles the writing of GOT entries // without dynamic relocations. static unsigned handleMipsTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c, uint64_t offset, int64_t addend, RelExpr expr) { if (expr == R_MIPS_TLSLD) { in.mipsGot->addTlsIndex(*c.file); c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } if (expr == R_MIPS_TLSGD) { in.mipsGot->addDynTlsEntry(*c.file, sym); c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } return 0; } // Notes about General Dynamic and Local Dynamic TLS models below. They may // require the generation of a pair of GOT entries that have associated dynamic // relocations. The pair of GOT entries created are of the form GOT[e0] Module // Index (Used to find pointer to TLS block at run-time) GOT[e1] Offset of // symbol in TLS block. // // Returns the number of relocations processed. template static unsigned handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c, typename ELFT::uint offset, int64_t addend, RelExpr expr) { if (!sym.isTls()) return 0; if (config->emachine == EM_MIPS) return handleMipsTlsRelocation(type, sym, c, offset, addend, expr); if (oneof( expr) && config->shared) { if (in.got->addDynTlsEntry(sym)) { uint64_t off = in.got->getGlobalDynOffset(sym); mainPart->relaDyn->addReloc( {target->tlsDescRel, in.got, off, !sym.isPreemptible, &sym, 0}); } if (expr != R_TLSDESC_CALL) c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } bool toExecRelax = !config->shared && config->emachine != EM_ARM && config->emachine != EM_HEXAGON && config->emachine != EM_RISCV; // If we are producing an executable and the symbol is non-preemptable, it // must be defined and the code sequence can be relaxed to use Local-Exec. // // ARM and RISC-V do not support any relaxations for TLS relocations, however, // we can omit the DTPMOD dynamic relocations and resolve them at link time // because them are always 1. This may be necessary for static linking as // DTPMOD may not be expected at load time. bool isLocalInExecutable = !sym.isPreemptible && !config->shared; // Local Dynamic is for access to module local TLS variables, while still // being suitable for being dynamically loaded via dlopen. GOT[e0] is the // module index, with a special value of 0 for the current module. GOT[e1] is // unused. There only needs to be one module index entry. if (oneof( expr)) { // Local-Dynamic relocs can be relaxed to Local-Exec. if (toExecRelax) { c.relocations.push_back( {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_LD_TO_LE), type, offset, addend, &sym}); return target->getTlsGdRelaxSkip(type); } if (expr == R_TLSLD_HINT) return 1; if (in.got->addTlsIndex()) { if (isLocalInExecutable) in.got->relocations.push_back( {R_ADDEND, target->symbolicRel, in.got->getTlsIndexOff(), 1, &sym}); else mainPart->relaDyn->addReloc(target->tlsModuleIndexRel, in.got, in.got->getTlsIndexOff(), nullptr); } c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } // Local-Dynamic relocs can be relaxed to Local-Exec. if (expr == R_DTPREL && toExecRelax) { c.relocations.push_back( {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_LD_TO_LE), type, offset, addend, &sym}); return 1; } // Local-Dynamic sequence where offset of tls variable relative to dynamic // thread pointer is stored in the got. This cannot be relaxed to Local-Exec. if (expr == R_TLSLD_GOT_OFF) { if (!sym.isInGot()) { in.got->addEntry(sym); uint64_t off = sym.getGotOffset(); in.got->relocations.push_back( {R_ABS, target->tlsOffsetRel, off, 0, &sym}); } c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } if (oneof(expr)) { if (!toExecRelax) { if (in.got->addDynTlsEntry(sym)) { uint64_t off = in.got->getGlobalDynOffset(sym); if (isLocalInExecutable) // Write one to the GOT slot. in.got->relocations.push_back( {R_ADDEND, target->symbolicRel, off, 1, &sym}); else mainPart->relaDyn->addReloc(target->tlsModuleIndexRel, in.got, off, &sym); // If the symbol is preemptible we need the dynamic linker to write // the offset too. uint64_t offsetOff = off + config->wordsize; if (sym.isPreemptible) mainPart->relaDyn->addReloc(target->tlsOffsetRel, in.got, offsetOff, &sym); else in.got->relocations.push_back( {R_ABS, target->tlsOffsetRel, offsetOff, 0, &sym}); } c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } // Global-Dynamic relocs can be relaxed to Initial-Exec or Local-Exec // depending on the symbol being locally defined or not. if (sym.isPreemptible) { c.relocations.push_back( {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_GD_TO_IE), type, offset, addend, &sym}); if (!sym.isInGot()) { in.got->addEntry(sym); mainPart->relaDyn->addReloc(target->tlsGotRel, in.got, sym.getGotOffset(), &sym); } } else { c.relocations.push_back( {target->adjustRelaxExpr(type, nullptr, R_RELAX_TLS_GD_TO_LE), type, offset, addend, &sym}); } return target->getTlsGdRelaxSkip(type); } // Initial-Exec relocs can be relaxed to Local-Exec if the symbol is locally // defined. if (oneof(expr) && toExecRelax && isLocalInExecutable) { c.relocations.push_back({R_RELAX_TLS_IE_TO_LE, type, offset, addend, &sym}); return 1; } if (expr == R_TLSIE_HINT) return 1; return 0; } static RelType getMipsPairType(RelType type, bool isLocal) { switch (type) { case R_MIPS_HI16: return R_MIPS_LO16; case R_MIPS_GOT16: // In case of global symbol, the R_MIPS_GOT16 relocation does not // have a pair. Each global symbol has a unique entry in the GOT // and a corresponding instruction with help of the R_MIPS_GOT16 // relocation loads an address of the symbol. In case of local // symbol, the R_MIPS_GOT16 relocation creates a GOT entry to hold // the high 16 bits of the symbol's value. A paired R_MIPS_LO16 // relocations handle low 16 bits of the address. That allows // to allocate only one GOT entry for every 64 KBytes of local data. return isLocal ? R_MIPS_LO16 : R_MIPS_NONE; case R_MICROMIPS_GOT16: return isLocal ? R_MICROMIPS_LO16 : R_MIPS_NONE; case R_MIPS_PCHI16: return R_MIPS_PCLO16; case R_MICROMIPS_HI16: return R_MICROMIPS_LO16; default: return R_MIPS_NONE; } } // True if non-preemptable symbol always has the same value regardless of where // the DSO is loaded. static bool isAbsolute(const Symbol &sym) { if (sym.isUndefWeak()) return true; if (const auto *dr = dyn_cast(&sym)) return dr->section == nullptr; // Absolute symbol. return false; } static bool isAbsoluteValue(const Symbol &sym) { return isAbsolute(sym) || sym.isTls(); } // Returns true if Expr refers a PLT entry. static bool needsPlt(RelExpr expr) { return oneof(expr); } // Returns true if Expr refers a GOT entry. Note that this function // returns false for TLS variables even though they need GOT, because // TLS variables uses GOT differently than the regular variables. static bool needsGot(RelExpr expr) { return oneof( expr); } // True if this expression is of the form Sym - X, where X is a position in the // file (PC, or GOT for example). static bool isRelExpr(RelExpr expr) { return oneof(expr); } // Returns true if a given relocation can be computed at link-time. // // For instance, we know the offset from a relocation to its target at // link-time if the relocation is PC-relative and refers a // non-interposable function in the same executable. This function // will return true for such relocation. // // If this function returns false, that means we need to emit a // dynamic relocation so that the relocation will be fixed at load-time. static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, InputSectionBase &s, uint64_t relOff) { // These expressions always compute a constant if (oneof( e)) return true; // These never do, except if the entire file is position dependent or if // only the low bits are used. if (e == R_GOT || e == R_PLT || e == R_TLSDESC) return target->usesOnlyLowPageBits(type) || !config->isPic; if (sym.isPreemptible) return false; if (!config->isPic) return true; // The size of a non preemptible symbol is a constant. if (e == R_SIZE) return true; // For the target and the relocation, we want to know if they are // absolute or relative. bool absVal = isAbsoluteValue(sym); bool relE = isRelExpr(e); if (absVal && !relE) return true; if (!absVal && relE) return true; if (!absVal && !relE) return target->usesOnlyLowPageBits(type); assert(absVal && relE); // Allow R_PLT_PC (optimized to R_PC here) to a hidden undefined weak symbol // in PIC mode. This is a little strange, but it allows us to link function // calls to such symbols (e.g. glibc/stdlib/exit.c:__run_exit_handlers). // Normally such a call will be guarded with a comparison, which will load a // zero from the GOT. if (sym.isUndefWeak()) return true; // We set the final symbols values for linker script defined symbols later. // They always can be computed as a link time constant. if (sym.scriptDefined) return true; error("relocation " + toString(type) + " cannot refer to absolute symbol: " + toString(sym) + getLocation(s, sym, relOff)); return true; } static RelExpr toPlt(RelExpr expr) { switch (expr) { case R_PPC64_CALL: return R_PPC64_CALL_PLT; case R_PC: return R_PLT_PC; case R_ABS: return R_PLT; default: return expr; } } static RelExpr fromPlt(RelExpr expr) { // We decided not to use a plt. Optimize a reference to the plt to a // reference to the symbol itself. switch (expr) { case R_PLT_PC: case R_PPC32_PLTREL: return R_PC; case R_PPC64_CALL_PLT: return R_PPC64_CALL; case R_PLT: return R_ABS; default: return expr; } } // Returns true if a given shared symbol is in a read-only segment in a DSO. template static bool isReadOnly(SharedSymbol &ss) { using Elf_Phdr = typename ELFT::Phdr; // Determine if the symbol is read-only by scanning the DSO's program headers. const SharedFile &file = ss.getFile(); for (const Elf_Phdr &phdr : check(file.template getObj().program_headers())) if ((phdr.p_type == ELF::PT_LOAD || phdr.p_type == ELF::PT_GNU_RELRO) && !(phdr.p_flags & ELF::PF_W) && ss.value >= phdr.p_vaddr && ss.value < phdr.p_vaddr + phdr.p_memsz) return true; return false; } // Returns symbols at the same offset as a given symbol, including SS itself. // // If two or more symbols are at the same offset, and at least one of // them are copied by a copy relocation, all of them need to be copied. // Otherwise, they would refer to different places at runtime. template static SmallSet getSymbolsAt(SharedSymbol &ss) { using Elf_Sym = typename ELFT::Sym; SharedFile &file = ss.getFile(); SmallSet ret; for (const Elf_Sym &s : file.template getGlobalELFSyms()) { if (s.st_shndx == SHN_UNDEF || s.st_shndx == SHN_ABS || s.getType() == STT_TLS || s.st_value != ss.value) continue; StringRef name = check(s.getName(file.getStringTable())); Symbol *sym = symtab->find(name); if (auto *alias = dyn_cast_or_null(sym)) ret.insert(alias); } return ret; } // When a symbol is copy relocated or we create a canonical plt entry, it is // effectively a defined symbol. In the case of copy relocation the symbol is // in .bss and in the case of a canonical plt entry it is in .plt. This function // replaces the existing symbol with a Defined pointing to the appropriate // location. static void replaceWithDefined(Symbol &sym, SectionBase *sec, uint64_t value, uint64_t size) { Symbol old = sym; sym.replace(Defined{sym.file, sym.getName(), sym.binding, sym.stOther, sym.type, value, size, sec}); sym.pltIndex = old.pltIndex; sym.gotIndex = old.gotIndex; sym.verdefIndex = old.verdefIndex; sym.exportDynamic = true; sym.isUsedInRegularObj = true; } // Reserve space in .bss or .bss.rel.ro for copy relocation. // // The copy relocation is pretty much a hack. If you use a copy relocation // in your program, not only the symbol name but the symbol's size, RW/RO // bit and alignment become part of the ABI. In addition to that, if the // symbol has aliases, the aliases become part of the ABI. That's subtle, // but if you violate that implicit ABI, that can cause very counter- // intuitive consequences. // // So, what is the copy relocation? It's for linking non-position // independent code to DSOs. In an ideal world, all references to data // exported by DSOs should go indirectly through GOT. But if object files // are compiled as non-PIC, all data references are direct. There is no // way for the linker to transform the code to use GOT, as machine // instructions are already set in stone in object files. This is where // the copy relocation takes a role. // // A copy relocation instructs the dynamic linker to copy data from a DSO // to a specified address (which is usually in .bss) at load-time. If the // static linker (that's us) finds a direct data reference to a DSO // symbol, it creates a copy relocation, so that the symbol can be // resolved as if it were in .bss rather than in a DSO. // // As you can see in this function, we create a copy relocation for the // dynamic linker, and the relocation contains not only symbol name but // various other information about the symbol. So, such attributes become a // part of the ABI. // // Note for application developers: I can give you a piece of advice if // you are writing a shared library. You probably should export only // functions from your library. You shouldn't export variables. // // As an example what can happen when you export variables without knowing // the semantics of copy relocations, assume that you have an exported // variable of type T. It is an ABI-breaking change to add new members at // end of T even though doing that doesn't change the layout of the // existing members. That's because the space for the new members are not // reserved in .bss unless you recompile the main program. That means they // are likely to overlap with other data that happens to be laid out next // to the variable in .bss. This kind of issue is sometimes very hard to // debug. What's a solution? Instead of exporting a variable V from a DSO, // define an accessor getV(). template static void addCopyRelSymbol(SharedSymbol &ss) { // Copy relocation against zero-sized symbol doesn't make sense. uint64_t symSize = ss.getSize(); if (symSize == 0 || ss.alignment == 0) fatal("cannot create a copy relocation for symbol " + toString(ss)); // See if this symbol is in a read-only segment. If so, preserve the symbol's // memory protection by reserving space in the .bss.rel.ro section. bool isRO = isReadOnly(ss); BssSection *sec = make(isRO ? ".bss.rel.ro" : ".bss", symSize, ss.alignment); OutputSection *osec = (isRO ? in.bssRelRo : in.bss)->getParent(); // At this point, sectionBases has been migrated to sections. Append sec to // sections. if (osec->sectionCommands.empty() || !isa(osec->sectionCommands.back())) osec->sectionCommands.push_back(make("")); auto *isd = cast(osec->sectionCommands.back()); isd->sections.push_back(sec); osec->commitSection(sec); // Look through the DSO's dynamic symbol table for aliases and create a // dynamic symbol for each one. This causes the copy relocation to correctly // interpose any aliases. for (SharedSymbol *sym : getSymbolsAt(ss)) replaceWithDefined(*sym, sec, 0, sym->size); mainPart->relaDyn->addReloc(target->copyRel, sec, 0, &ss); } // MIPS has an odd notion of "paired" relocations to calculate addends. // For example, if a relocation is of R_MIPS_HI16, there must be a // R_MIPS_LO16 relocation after that, and an addend is calculated using // the two relocations. template static int64_t computeMipsAddend(const RelTy &rel, const RelTy *end, InputSectionBase &sec, RelExpr expr, bool isLocal) { if (expr == R_MIPS_GOTREL && isLocal) return sec.getFile()->mipsGp0; // The ABI says that the paired relocation is used only for REL. // See p. 4-17 at ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf if (RelTy::IsRela) return 0; RelType type = rel.getType(config->isMips64EL); uint32_t pairTy = getMipsPairType(type, isLocal); if (pairTy == R_MIPS_NONE) return 0; const uint8_t *buf = sec.data().data(); uint32_t symIndex = rel.getSymbol(config->isMips64EL); // To make things worse, paired relocations might not be contiguous in // the relocation table, so we need to do linear search. *sigh* for (const RelTy *ri = &rel; ri != end; ++ri) if (ri->getType(config->isMips64EL) == pairTy && ri->getSymbol(config->isMips64EL) == symIndex) return target->getImplicitAddend(buf + ri->r_offset, pairTy); warn("can't find matching " + toString(pairTy) + " relocation for " + toString(type)); return 0; } // Returns an addend of a given relocation. If it is RELA, an addend // is in a relocation itself. If it is REL, we need to read it from an // input section. template static int64_t computeAddend(const RelTy &rel, const RelTy *end, InputSectionBase &sec, RelExpr expr, bool isLocal) { int64_t addend; RelType type = rel.getType(config->isMips64EL); if (RelTy::IsRela) { addend = getAddend(rel); } else { const uint8_t *buf = sec.data().data(); addend = target->getImplicitAddend(buf + rel.r_offset, type); } if (config->emachine == EM_PPC64 && config->isPic && type == R_PPC64_TOC) addend += getPPC64TocBase(); if (config->emachine == EM_MIPS) addend += computeMipsAddend(rel, end, sec, expr, isLocal); return addend; } // Custom error message if Sym is defined in a discarded section. template static std::string maybeReportDiscarded(Undefined &sym) { auto *file = dyn_cast_or_null>(sym.file); if (!file || !sym.discardedSecIdx || file->getSections()[sym.discardedSecIdx] != &InputSection::discarded) return ""; ArrayRef> objSections = CHECK(file->getObj().sections(), file); std::string msg; if (sym.type == ELF::STT_SECTION) { msg = "relocation refers to a discarded section: "; msg += CHECK( file->getObj().getSectionName(objSections[sym.discardedSecIdx]), file); } else { msg = "relocation refers to a symbol in a discarded section: " + toString(sym); } msg += "\n>>> defined in " + toString(file); Elf_Shdr_Impl elfSec = objSections[sym.discardedSecIdx - 1]; if (elfSec.sh_type != SHT_GROUP) return msg; // If the discarded section is a COMDAT. StringRef signature = file->getShtGroupSignature(objSections, elfSec); if (const InputFile *prevailing = symtab->comdatGroups.lookup(CachedHashStringRef(signature))) msg += "\n>>> section group signature: " + signature.str() + "\n>>> prevailing definition is in " + toString(prevailing); return msg; } // Undefined diagnostics are collected in a vector and emitted once all of // them are known, so that some postprocessing on the list of undefined symbols // can happen before lld emits diagnostics. struct UndefinedDiag { Symbol *sym; struct Loc { InputSectionBase *sec; uint64_t offset; }; std::vector locs; bool isWarning; }; static std::vector undefs; // Check whether the definition name def is a mangled function name that matches // the reference name ref. static bool canSuggestExternCForCXX(StringRef ref, StringRef def) { llvm::ItaniumPartialDemangler d; std::string name = def.str(); if (d.partialDemangle(name.c_str())) return false; char *buf = d.getFunctionName(nullptr, nullptr); if (!buf) return false; bool ret = ref == buf; free(buf); return ret; } // Suggest an alternative spelling of an "undefined symbol" diagnostic. Returns // the suggested symbol, which is either in the symbol table, or in the same // file of sym. template static const Symbol *getAlternativeSpelling(const Undefined &sym, std::string &pre_hint, std::string &post_hint) { DenseMap map; if (auto *file = dyn_cast_or_null>(sym.file)) { // If sym is a symbol defined in a discarded section, maybeReportDiscarded() // will give an error. Don't suggest an alternative spelling. if (file && sym.discardedSecIdx != 0 && file->getSections()[sym.discardedSecIdx] == &InputSection::discarded) return nullptr; // Build a map of local defined symbols. for (const Symbol *s : sym.file->getSymbols()) if (s->isLocal() && s->isDefined()) map.try_emplace(s->getName(), s); } auto suggest = [&](StringRef newName) -> const Symbol * { // If defined locally. if (const Symbol *s = map.lookup(newName)) return s; // If in the symbol table and not undefined. if (const Symbol *s = symtab->find(newName)) if (!s->isUndefined()) return s; return nullptr; }; // This loop enumerates all strings of Levenshtein distance 1 as typo // correction candidates and suggests the one that exists as a non-undefined // symbol. StringRef name = sym.getName(); for (size_t i = 0, e = name.size(); i != e + 1; ++i) { // Insert a character before name[i]. std::string newName = (name.substr(0, i) + "0" + name.substr(i)).str(); for (char c = '0'; c <= 'z'; ++c) { newName[i] = c; if (const Symbol *s = suggest(newName)) return s; } if (i == e) break; // Substitute name[i]. newName = std::string(name); for (char c = '0'; c <= 'z'; ++c) { newName[i] = c; if (const Symbol *s = suggest(newName)) return s; } // Transpose name[i] and name[i+1]. This is of edit distance 2 but it is // common. if (i + 1 < e) { newName[i] = name[i + 1]; newName[i + 1] = name[i]; if (const Symbol *s = suggest(newName)) return s; } // Delete name[i]. newName = (name.substr(0, i) + name.substr(i + 1)).str(); if (const Symbol *s = suggest(newName)) return s; } // Case mismatch, e.g. Foo vs FOO. for (auto &it : map) if (name.equals_lower(it.first)) return it.second; for (Symbol *sym : symtab->symbols()) if (!sym->isUndefined() && name.equals_lower(sym->getName())) return sym; // The reference may be a mangled name while the definition is not. Suggest a // missing extern "C". if (name.startswith("_Z")) { std::string buf = name.str(); llvm::ItaniumPartialDemangler d; if (!d.partialDemangle(buf.c_str())) if (char *buf = d.getFunctionName(nullptr, nullptr)) { const Symbol *s = suggest(buf); free(buf); if (s) { pre_hint = ": extern \"C\" "; return s; } } } else { const Symbol *s = nullptr; for (auto &it : map) if (canSuggestExternCForCXX(name, it.first)) { s = it.second; break; } if (!s) for (Symbol *sym : symtab->symbols()) if (canSuggestExternCForCXX(name, sym->getName())) { s = sym; break; } if (s) { pre_hint = " to declare "; post_hint = " as extern \"C\"?"; return s; } } return nullptr; } template static void reportUndefinedSymbol(const UndefinedDiag &undef, bool correctSpelling) { Symbol &sym = *undef.sym; auto visibility = [&]() -> std::string { switch (sym.visibility) { case STV_INTERNAL: return "internal "; case STV_HIDDEN: return "hidden "; case STV_PROTECTED: return "protected "; default: return ""; } }; std::string msg = maybeReportDiscarded(cast(sym)); if (msg.empty()) msg = "undefined " + visibility() + "symbol: " + toString(sym); const size_t maxUndefReferences = 3; size_t i = 0; for (UndefinedDiag::Loc l : undef.locs) { if (i >= maxUndefReferences) break; InputSectionBase &sec = *l.sec; uint64_t offset = l.offset; msg += "\n>>> referenced by "; std::string src = sec.getSrcMsg(sym, offset); if (!src.empty()) msg += src + "\n>>> "; msg += sec.getObjMsg(offset); i++; } if (i < undef.locs.size()) msg += ("\n>>> referenced " + Twine(undef.locs.size() - i) + " more times") .str(); if (correctSpelling) { std::string pre_hint = ": ", post_hint; if (const Symbol *corrected = getAlternativeSpelling( cast(sym), pre_hint, post_hint)) { msg += "\n>>> did you mean" + pre_hint + toString(*corrected) + post_hint; if (corrected->file) msg += "\n>>> defined in: " + toString(corrected->file); } } if (sym.getName().startswith("_ZTV")) msg += "\n>>> the vtable symbol may be undefined because the class is missing " "its key function (see https://lld.llvm.org/missingkeyfunction)"; if (undef.isWarning) warn(msg); else error(msg); } template void elf::reportUndefinedSymbols() { // Find the first "undefined symbol" diagnostic for each diagnostic, and // collect all "referenced from" lines at the first diagnostic. DenseMap firstRef; for (UndefinedDiag &undef : undefs) { assert(undef.locs.size() == 1); if (UndefinedDiag *canon = firstRef.lookup(undef.sym)) { canon->locs.push_back(undef.locs[0]); undef.locs.clear(); } else firstRef[undef.sym] = &undef; } // Enable spell corrector for the first 2 diagnostics. for (auto it : enumerate(undefs)) if (!it.value().locs.empty()) reportUndefinedSymbol(it.value(), it.index() < 2); undefs.clear(); } // Report an undefined symbol if necessary. // Returns true if the undefined symbol will produce an error message. static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec, uint64_t offset) { if (!sym.isUndefined() || sym.isWeak()) return false; bool canBeExternal = !sym.isLocal() && sym.visibility == STV_DEFAULT; if (config->unresolvedSymbols == UnresolvedPolicy::Ignore && canBeExternal) return false; // clang (as of 2019-06-12) / gcc (as of 8.2.1) PPC64 may emit a .rela.toc // which references a switch table in a discarded .rodata/.text section. The // .toc and the .rela.toc are incorrectly not placed in the comdat. The ELF // spec says references from outside the group to a STB_LOCAL symbol are not // allowed. Work around the bug. // // PPC32 .got2 is similar but cannot be fixed. Multiple .got2 is infeasible // because .LC0-.LTOC is not representable if the two labels are in different // .got2 if (cast(sym).discardedSecIdx != 0 && (sec.name == ".got2" || sec.name == ".toc")) return false; bool isWarning = (config->unresolvedSymbols == UnresolvedPolicy::Warn && canBeExternal) || config->noinhibitExec; undefs.push_back({&sym, {{&sec, offset}}, isWarning}); return !isWarning; } // MIPS N32 ABI treats series of successive relocations with the same offset // as a single relocation. The similar approach used by N64 ABI, but this ABI // packs all relocations into the single relocation record. Here we emulate // this for the N32 ABI. Iterate over relocation with the same offset and put // theirs types into the single bit-set. template static RelType getMipsN32RelType(RelTy *&rel, RelTy *end) { RelType type = 0; uint64_t offset = rel->r_offset; int n = 0; while (rel != end && rel->r_offset == offset) type |= (rel++)->getType(config->isMips64EL) << (8 * n++); return type; } // .eh_frame sections are mergeable input sections, so their input // offsets are not linearly mapped to output section. For each input // offset, we need to find a section piece containing the offset and // add the piece's base address to the input offset to compute the // output offset. That isn't cheap. // // This class is to speed up the offset computation. When we process // relocations, we access offsets in the monotonically increasing // order. So we can optimize for that access pattern. // // For sections other than .eh_frame, this class doesn't do anything. namespace { class OffsetGetter { public: explicit OffsetGetter(InputSectionBase &sec) { if (auto *eh = dyn_cast(&sec)) pieces = eh->pieces; } // Translates offsets in input sections to offsets in output sections. // Given offset must increase monotonically. We assume that Piece is // sorted by inputOff. uint64_t get(uint64_t off) { if (pieces.empty()) return off; while (i != pieces.size() && pieces[i].inputOff + pieces[i].size <= off) ++i; if (i == pieces.size()) fatal(".eh_frame: relocation is not in any piece"); // Pieces must be contiguous, so there must be no holes in between. assert(pieces[i].inputOff <= off && "Relocation not in any piece"); // Offset -1 means that the piece is dead (i.e. garbage collected). if (pieces[i].outputOff == -1) return -1; return pieces[i].outputOff + off - pieces[i].inputOff; } private: ArrayRef pieces; size_t i = 0; }; } // namespace static void addRelativeReloc(InputSectionBase *isec, uint64_t offsetInSec, Symbol *sym, int64_t addend, RelExpr expr, RelType type) { Partition &part = isec->getPartition(); // Add a relative relocation. If relrDyn section is enabled, and the // relocation offset is guaranteed to be even, add the relocation to // the relrDyn section, otherwise add it to the relaDyn section. // relrDyn sections don't support odd offsets. Also, relrDyn sections // don't store the addend values, so we must write it to the relocated // address. if (part.relrDyn && isec->alignment >= 2 && offsetInSec % 2 == 0) { isec->relocations.push_back({expr, type, offsetInSec, addend, sym}); part.relrDyn->relocs.push_back({isec, offsetInSec}); return; } part.relaDyn->addReloc(target->relativeRel, isec, offsetInSec, sym, addend, expr, type); } template static void addPltEntry(PltSection *plt, GotPltSection *gotPlt, RelocationBaseSection *rel, RelType type, Symbol &sym) { plt->addEntry(sym); gotPlt->addEntry(sym); rel->addReloc( {type, gotPlt, sym.getGotPltOffset(), !sym.isPreemptible, &sym, 0}); } static void addGotEntry(Symbol &sym) { in.got->addEntry(sym); RelExpr expr = sym.isTls() ? R_TLS : R_ABS; uint64_t off = sym.getGotOffset(); // If a GOT slot value can be calculated at link-time, which is now, // we can just fill that out. // // (We don't actually write a value to a GOT slot right now, but we // add a static relocation to a Relocations vector so that // InputSection::relocate will do the work for us. We may be able // to just write a value now, but it is a TODO.) bool isLinkTimeConstant = !sym.isPreemptible && (!config->isPic || isAbsolute(sym)); if (isLinkTimeConstant) { in.got->relocations.push_back({expr, target->symbolicRel, off, 0, &sym}); return; } // Otherwise, we emit a dynamic relocation to .rel[a].dyn so that // the GOT slot will be fixed at load-time. if (!sym.isTls() && !sym.isPreemptible && config->isPic && !isAbsolute(sym)) { addRelativeReloc(in.got, off, &sym, 0, R_ABS, target->symbolicRel); return; } mainPart->relaDyn->addReloc( sym.isTls() ? target->tlsGotRel : target->gotRel, in.got, off, &sym, 0, sym.isPreemptible ? R_ADDEND : R_ABS, target->symbolicRel); } // Return true if we can define a symbol in the executable that // contains the value/function of a symbol defined in a shared // library. static bool canDefineSymbolInExecutable(Symbol &sym) { // If the symbol has default visibility the symbol defined in the // executable will preempt it. // Note that we want the visibility of the shared symbol itself, not // the visibility of the symbol in the output file we are producing. That is // why we use Sym.stOther. if ((sym.stOther & 0x3) == STV_DEFAULT) return true; // If we are allowed to break address equality of functions, defining // a plt entry will allow the program to call the function in the // .so, but the .so and the executable will no agree on the address // of the function. Similar logic for objects. return ((sym.isFunc() && config->ignoreFunctionAddressEquality) || (sym.isObject() && config->ignoreDataAddressEquality)); } // The reason we have to do this early scan is as follows // * To mmap the output file, we need to know the size // * For that, we need to know how many dynamic relocs we will have. // It might be possible to avoid this by outputting the file with write: // * Write the allocated output sections, computing addresses. // * Apply relocations, recording which ones require a dynamic reloc. // * Write the dynamic relocations. // * Write the rest of the file. // This would have some drawbacks. For example, we would only know if .rela.dyn // is needed after applying relocations. If it is, it will go after rw and rx // sections. Given that it is ro, we will need an extra PT_LOAD. This // complicates things for the dynamic linker and means we would have to reserve // space for the extra PT_LOAD even if we end up not using it. template static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, uint64_t offset, Symbol &sym, const RelTy &rel, int64_t addend) { // If the relocation is known to be a link-time constant, we know no dynamic // relocation will be created, pass the control to relocateAlloc() or // relocateNonAlloc() to resolve it. // // The behavior of an undefined weak reference is implementation defined. If // the relocation is to a weak undef, and we are producing an executable, let // relocate{,Non}Alloc() resolve it. if (isStaticLinkTimeConstant(expr, type, sym, sec, offset) || (!config->shared && sym.isUndefWeak())) { sec.relocations.push_back({expr, type, offset, addend, &sym}); return; } bool canWrite = (sec.flags & SHF_WRITE) || !config->zText; if (canWrite) { RelType rel = target->getDynRel(type); if (expr == R_GOT || (rel == target->symbolicRel && !sym.isPreemptible)) { addRelativeReloc(&sec, offset, &sym, addend, expr, type); return; } else if (rel != 0) { if (config->emachine == EM_MIPS && rel == target->symbolicRel) rel = target->relativeRel; sec.getPartition().relaDyn->addReloc(rel, &sec, offset, &sym, addend, R_ADDEND, type); // MIPS ABI turns using of GOT and dynamic relocations inside out. // While regular ABI uses dynamic relocations to fill up GOT entries // MIPS ABI requires dynamic linker to fills up GOT entries using // specially sorted dynamic symbol table. This affects even dynamic // relocations against symbols which do not require GOT entries // creation explicitly, i.e. do not have any GOT-relocations. So if // a preemptible symbol has a dynamic relocation we anyway have // to create a GOT entry for it. // If a non-preemptible symbol has a dynamic relocation against it, // dynamic linker takes it st_value, adds offset and writes down // result of the dynamic relocation. In case of preemptible symbol // dynamic linker performs symbol resolution, writes the symbol value // to the GOT entry and reads the GOT entry when it needs to perform // a dynamic relocation. // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf p.4-19 if (config->emachine == EM_MIPS) in.mipsGot->addEntry(*sec.file, sym, addend, expr); return; } } // When producing an executable, we can perform copy relocations (for // STT_OBJECT) and canonical PLT (for STT_FUNC). if (!config->shared) { if (!canDefineSymbolInExecutable(sym)) { errorOrWarn("cannot preempt symbol: " + toString(sym) + getLocation(sec, sym, offset)); return; } if (sym.isObject()) { // Produce a copy relocation. if (auto *ss = dyn_cast(&sym)) { if (!config->zCopyreloc) error("unresolvable relocation " + toString(type) + " against symbol '" + toString(*ss) + "'; recompile with -fPIC or remove '-z nocopyreloc'" + getLocation(sec, sym, offset)); addCopyRelSymbol(*ss); } sec.relocations.push_back({expr, type, offset, addend, &sym}); return; } // This handles a non PIC program call to function in a shared library. In // an ideal world, we could just report an error saying the relocation can // overflow at runtime. In the real world with glibc, crt1.o has a // R_X86_64_PC32 pointing to libc.so. // // The general idea on how to handle such cases is to create a PLT entry and // use that as the function value. // // For the static linking part, we just return a plt expr and everything // else will use the PLT entry as the address. // // The remaining problem is making sure pointer equality still works. We // need the help of the dynamic linker for that. We let it know that we have // a direct reference to a so symbol by creating an undefined symbol with a // non zero st_value. Seeing that, the dynamic linker resolves the symbol to // the value of the symbol we created. This is true even for got entries, so // pointer equality is maintained. To avoid an infinite loop, the only entry // that points to the real function is a dedicated got entry used by the // plt. That is identified by special relocation types (R_X86_64_JUMP_SLOT, // R_386_JMP_SLOT, etc). // For position independent executable on i386, the plt entry requires ebx // to be set. This causes two problems: // * If some code has a direct reference to a function, it was probably // compiled without -fPIE/-fPIC and doesn't maintain ebx. // * If a library definition gets preempted to the executable, it will have // the wrong ebx value. if (sym.isFunc()) { if (config->pie && config->emachine == EM_386) errorOrWarn("symbol '" + toString(sym) + "' cannot be preempted; recompile with -fPIE" + getLocation(sec, sym, offset)); if (!sym.isInPlt()) addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym); if (!sym.isDefined()) { replaceWithDefined( sym, in.plt, target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0); if (config->emachine == EM_PPC) { // PPC32 canonical PLT entries are at the beginning of .glink cast(sym).value = in.plt->headerSize; in.plt->headerSize += 16; cast(in.plt)->canonical_plts.push_back(&sym); } } sym.needsPltAddr = true; sec.relocations.push_back({expr, type, offset, addend, &sym}); return; } } if (config->isPic) { if (!canWrite && !isRelExpr(expr)) errorOrWarn( "can't create dynamic relocation " + toString(type) + " against " + (sym.getName().empty() ? "local symbol" : "symbol: " + toString(sym)) + " in readonly segment; recompile object files with -fPIC " "or pass '-Wl,-z,notext' to allow text relocations in the output" + getLocation(sec, sym, offset)); else errorOrWarn( "relocation " + toString(type) + " cannot be used against " + (sym.getName().empty() ? "local symbol" : "symbol " + toString(sym)) + "; recompile with -fPIC" + getLocation(sec, sym, offset)); return; } errorOrWarn("symbol '" + toString(sym) + "' has no type" + getLocation(sec, sym, offset)); } template static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, RelTy *start, RelTy *end) { const RelTy &rel = *i; uint32_t symIndex = rel.getSymbol(config->isMips64EL); Symbol &sym = sec.getFile()->getSymbol(symIndex); RelType type; // Deal with MIPS oddity. if (config->mipsN32Abi) { type = getMipsN32RelType(i, end); } else { type = rel.getType(config->isMips64EL); ++i; } // Get an offset in an output section this relocation is applied to. uint64_t offset = getOffset.get(rel.r_offset); if (offset == uint64_t(-1)) return; // Error if the target symbol is undefined. Symbol index 0 may be used by // marker relocations, e.g. R_*_NONE and R_ARM_V4BX. Don't error on them. if (symIndex != 0 && maybeReportUndefined(sym, sec, rel.r_offset)) return; const uint8_t *relocatedAddr = sec.data().begin() + rel.r_offset; RelExpr expr = target->getRelExpr(type, sym, relocatedAddr); // Ignore R_*_NONE and other marker relocations. if (expr == R_NONE) return; if (sym.isGnuIFunc() && !config->zText && config->warnIfuncTextrel) { warn("using ifunc symbols when text relocations are allowed may produce " "a binary that will segfault, if the object file is linked with " "old version of glibc (glibc 2.28 and earlier). If this applies to " "you, consider recompiling the object files without -fPIC and " "without -Wl,-z,notext option. Use -no-warn-ifunc-textrel to " "turn off this warning." + getLocation(sec, sym, offset)); } // Read an addend. int64_t addend = computeAddend(rel, end, sec, expr, sym.isLocal()); if (config->emachine == EM_PPC64) { // For a call to __tls_get_addr, the instruction needs to be relocated by // two relocations, R_PPC64_TLSGD/R_PPC64_TLSLD and R_PPC64_REL24[_NOTOC]. // R_PPC64_TLSGD/R_PPC64_TLSLD should precede R_PPC64_REL24[_NOTOC]. if ((type == R_PPC64_REL24 || type == R_PPC64_REL24_NOTOC) && sym.getName() == "__tls_get_addr") { bool err = i - start < 2; if (!err) { // Subtract 2 to get the previous iterator because we have already done // ++i above. This is now safe because we know that i-1 is not the // start. const RelTy &prevRel = *(i - 2); RelType prevType = prevRel.getType(config->isMips64EL); err = prevRel.r_offset != rel.r_offset || (prevType != R_PPC64_TLSGD && prevType != R_PPC64_TLSLD); } if (err) errorOrWarn("call to __tls_get_addr is missing a " "R_PPC64_TLSGD/R_PPC64_TLSLD relocation" + getLocation(sec, sym, offset)); } // We can separate the small code model relocations into 2 categories: // 1) Those that access the compiler generated .toc sections. // 2) Those that access the linker allocated got entries. // lld allocates got entries to symbols on demand. Since we don't try to // sort the got entries in any way, we don't have to track which objects // have got-based small code model relocs. The .toc sections get placed // after the end of the linker allocated .got section and we do sort those // so sections addressed with small code model relocations come first. if (isPPC64SmallCodeModelTocReloc(type)) sec.file->ppc64SmallCodeModelTocRelocs = true; // Record the TOC entry (.toc + addend) as not relaxable. See the comment in // InputSectionBase::relocateAlloc(). if (type == R_PPC64_TOC16_LO && sym.isSection() && isa(sym) && cast(sym).section->name == ".toc") ppc64noTocRelax.insert({&sym, addend}); - - if (type == R_PPC64_TLSGD && expr == R_TLSDESC_CALL) { - if (i == end) { - errorOrWarn("R_PPC64_TLSGD may not be the last relocation" + - getLocation(sec, sym, offset)); - return; - } - - // Offset the 4-byte aligned R_PPC64_TLSGD by one byte in the NOTOC case, - // so we can discern it later from the toc-case. - if (i->getType(/*isMips64EL=*/false) == R_PPC64_REL24_NOTOC) - ++offset; - } } // Relax relocations. // // If we know that a PLT entry will be resolved within the same ELF module, we // can skip PLT access and directly jump to the destination function. For // example, if we are linking a main executable, all dynamic symbols that can // be resolved within the executable will actually be resolved that way at // runtime, because the main executable is always at the beginning of a search // list. We can leverage that fact. if (!sym.isPreemptible && (!sym.isGnuIFunc() || config->zIfuncNoplt)) { if (expr == R_GOT_PC && !isAbsoluteValue(sym)) { expr = target->adjustRelaxExpr(type, relocatedAddr, expr); } else { // The 0x8000 bit of r_addend of R_PPC_PLTREL24 is used to choose call // stub type. It should be ignored if optimized to R_PC. if (config->emachine == EM_PPC && expr == R_PPC32_PLTREL) addend &= ~0x8000; // R_HEX_GD_PLT_B22_PCREL (call a@GDPLT) is transformed into // call __tls_get_addr even if the symbol is non-preemptible. if (!(config->emachine == EM_HEXAGON && (type == R_HEX_GD_PLT_B22_PCREL || type == R_HEX_GD_PLT_B22_PCREL_X || type == R_HEX_GD_PLT_B32_PCREL_X))) expr = fromPlt(expr); } } // If the relocation does not emit a GOT or GOTPLT entry but its computation // uses their addresses, we need GOT or GOTPLT to be created. // // The 4 types that relative GOTPLT are all x86 and x86-64 specific. if (oneof(expr)) { in.gotPlt->hasGotPltOffRel = true; } else if (oneof( expr)) { in.got->hasGotOffRel = true; } // Process some TLS relocations, including relaxing TLS relocations. // Note that this function does not handle all TLS relocations. if (unsigned processed = handleTlsRelocation(type, sym, sec, offset, addend, expr)) { i += (processed - 1); return; } // We were asked not to generate PLT entries for ifuncs. Instead, pass the // direct relocation on through. if (sym.isGnuIFunc() && config->zIfuncNoplt) { sym.exportDynamic = true; mainPart->relaDyn->addReloc(type, &sec, offset, &sym, addend, R_ADDEND, type); return; } // Non-preemptible ifuncs require special handling. First, handle the usual // case where the symbol isn't one of these. if (!sym.isGnuIFunc() || sym.isPreemptible) { // If a relocation needs PLT, we create PLT and GOTPLT slots for the symbol. if (needsPlt(expr) && !sym.isInPlt()) addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym); // Create a GOT slot if a relocation needs GOT. if (needsGot(expr)) { if (config->emachine == EM_MIPS) { // MIPS ABI has special rules to process GOT entries and doesn't // require relocation entries for them. A special case is TLS // relocations. In that case dynamic loader applies dynamic // relocations to initialize TLS GOT entries. // See "Global Offset Table" in Chapter 5 in the following document // for detailed description: // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf in.mipsGot->addEntry(*sec.file, sym, addend, expr); } else if (!sym.isInGot()) { addGotEntry(sym); } } } else { // Handle a reference to a non-preemptible ifunc. These are special in a // few ways: // // - Unlike most non-preemptible symbols, non-preemptible ifuncs do not have // a fixed value. But assuming that all references to the ifunc are // GOT-generating or PLT-generating, the handling of an ifunc is // relatively straightforward. We create a PLT entry in Iplt, which is // usually at the end of .plt, which makes an indirect call using a // matching GOT entry in igotPlt, which is usually at the end of .got.plt. // The GOT entry is relocated using an IRELATIVE relocation in relaIplt, // which is usually at the end of .rela.plt. Unlike most relocations in // .rela.plt, which may be evaluated lazily without -z now, dynamic // loaders evaluate IRELATIVE relocs eagerly, which means that for // IRELATIVE relocs only, GOT-generating relocations can point directly to // .got.plt without requiring a separate GOT entry. // // - Despite the fact that an ifunc does not have a fixed value, compilers // that are not passed -fPIC will assume that they do, and will emit // direct (non-GOT-generating, non-PLT-generating) relocations to the // symbol. This means that if a direct relocation to the symbol is // seen, the linker must set a value for the symbol, and this value must // be consistent no matter what type of reference is made to the symbol. // This can be done by creating a PLT entry for the symbol in the way // described above and making it canonical, that is, making all references // point to the PLT entry instead of the resolver. In lld we also store // the address of the PLT entry in the dynamic symbol table, which means // that the symbol will also have the same value in other modules. // Because the value loaded from the GOT needs to be consistent with // the value computed using a direct relocation, a non-preemptible ifunc // may end up with two GOT entries, one in .got.plt that points to the // address returned by the resolver and is used only by the PLT entry, // and another in .got that points to the PLT entry and is used by // GOT-generating relocations. // // - The fact that these symbols do not have a fixed value makes them an // exception to the general rule that a statically linked executable does // not require any form of dynamic relocation. To handle these relocations // correctly, the IRELATIVE relocations are stored in an array which a // statically linked executable's startup code must enumerate using the // linker-defined symbols __rela?_iplt_{start,end}. if (!sym.isInPlt()) { // Create PLT and GOTPLT slots for the symbol. sym.isInIplt = true; // Create a copy of the symbol to use as the target of the IRELATIVE // relocation in the igotPlt. This is in case we make the PLT canonical // later, which would overwrite the original symbol. // // FIXME: Creating a copy of the symbol here is a bit of a hack. All // that's really needed to create the IRELATIVE is the section and value, // so ideally we should just need to copy those. auto *directSym = make(cast(sym)); addPltEntry(in.iplt, in.igotPlt, in.relaIplt, target->iRelativeRel, *directSym); sym.pltIndex = directSym->pltIndex; } if (needsGot(expr)) { // Redirect GOT accesses to point to the Igot. // // This field is also used to keep track of whether we ever needed a GOT // entry. If we did and we make the PLT canonical later, we'll need to // create a GOT entry pointing to the PLT entry for Sym. sym.gotInIgot = true; } else if (!needsPlt(expr)) { // Make the ifunc's PLT entry canonical by changing the value of its // symbol to redirect all references to point to it. auto &d = cast(sym); d.section = in.iplt; d.value = sym.pltIndex * target->ipltEntrySize; d.size = 0; // It's important to set the symbol type here so that dynamic loaders // don't try to call the PLT as if it were an ifunc resolver. d.type = STT_FUNC; if (sym.gotInIgot) { // We previously encountered a GOT generating reference that we // redirected to the Igot. Now that the PLT entry is canonical we must // clear the redirection to the Igot and add a GOT entry. As we've // changed the symbol type to STT_FUNC future GOT generating references // will naturally use this GOT entry. // // We don't need to worry about creating a MIPS GOT here because ifuncs // aren't a thing on MIPS. sym.gotInIgot = false; addGotEntry(sym); } } } processRelocAux(sec, expr, type, offset, sym, rel, addend); } template static void scanRelocs(InputSectionBase &sec, ArrayRef rels) { OffsetGetter getOffset(sec); // Not all relocations end up in Sec.Relocations, but a lot do. sec.relocations.reserve(rels.size()); for (auto i = rels.begin(), end = rels.end(); i != end;) scanReloc(sec, getOffset, i, rels.begin(), end); // Sort relocations by offset for more efficient searching for // R_RISCV_PCREL_HI20 and R_PPC64_ADDR64. if (config->emachine == EM_RISCV || (config->emachine == EM_PPC64 && sec.name == ".toc")) llvm::stable_sort(sec.relocations, [](const Relocation &lhs, const Relocation &rhs) { return lhs.offset < rhs.offset; }); } template void elf::scanRelocations(InputSectionBase &s) { if (s.areRelocsRela) scanRelocs(s, s.relas()); else scanRelocs(s, s.rels()); } static bool mergeCmp(const InputSection *a, const InputSection *b) { // std::merge requires a strict weak ordering. if (a->outSecOff < b->outSecOff) return true; if (a->outSecOff == b->outSecOff) { auto *ta = dyn_cast(a); auto *tb = dyn_cast(b); // Check if Thunk is immediately before any specific Target // InputSection for example Mips LA25 Thunks. if (ta && ta->getTargetInputSection() == b) return true; // Place Thunk Sections without specific targets before // non-Thunk Sections. if (ta && !tb && !ta->getTargetInputSection()) return true; } return false; } // Call Fn on every executable InputSection accessed via the linker script // InputSectionDescription::Sections. static void forEachInputSectionDescription( ArrayRef outputSections, llvm::function_ref fn) { for (OutputSection *os : outputSections) { if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR)) continue; for (BaseCommand *bc : os->sectionCommands) if (auto *isd = dyn_cast(bc)) fn(os, isd); } } // Thunk Implementation // // Thunks (sometimes called stubs, veneers or branch islands) are small pieces // of code that the linker inserts inbetween a caller and a callee. The thunks // are added at link time rather than compile time as the decision on whether // a thunk is needed, such as the caller and callee being out of range, can only // be made at link time. // // It is straightforward to tell given the current state of the program when a // thunk is needed for a particular call. The more difficult part is that // the thunk needs to be placed in the program such that the caller can reach // the thunk and the thunk can reach the callee; furthermore, adding thunks to // the program alters addresses, which can mean more thunks etc. // // In lld we have a synthetic ThunkSection that can hold many Thunks. // The decision to have a ThunkSection act as a container means that we can // more easily handle the most common case of a single block of contiguous // Thunks by inserting just a single ThunkSection. // // The implementation of Thunks in lld is split across these areas // Relocations.cpp : Framework for creating and placing thunks // Thunks.cpp : The code generated for each supported thunk // Target.cpp : Target specific hooks that the framework uses to decide when // a thunk is used // Synthetic.cpp : Implementation of ThunkSection // Writer.cpp : Iteratively call framework until no more Thunks added // // Thunk placement requirements: // Mips LA25 thunks. These must be placed immediately before the callee section // We can assume that the caller is in range of the Thunk. These are modelled // by Thunks that return the section they must precede with // getTargetInputSection(). // // ARM interworking and range extension thunks. These thunks must be placed // within range of the caller. All implemented ARM thunks can always reach the // callee as they use an indirect jump via a register that has no range // restrictions. // // Thunk placement algorithm: // For Mips LA25 ThunkSections; the placement is explicit, it has to be before // getTargetInputSection(). // // For thunks that must be placed within range of the caller there are many // possible choices given that the maximum range from the caller is usually // much larger than the average InputSection size. Desirable properties include: // - Maximize reuse of thunks by multiple callers // - Minimize number of ThunkSections to simplify insertion // - Handle impact of already added Thunks on addresses // - Simple to understand and implement // // In lld for the first pass, we pre-create one or more ThunkSections per // InputSectionDescription at Target specific intervals. A ThunkSection is // placed so that the estimated end of the ThunkSection is within range of the // start of the InputSectionDescription or the previous ThunkSection. For // example: // InputSectionDescription // Section 0 // ... // Section N // ThunkSection 0 // Section N + 1 // ... // Section N + K // Thunk Section 1 // // The intention is that we can add a Thunk to a ThunkSection that is well // spaced enough to service a number of callers without having to do a lot // of work. An important principle is that it is not an error if a Thunk cannot // be placed in a pre-created ThunkSection; when this happens we create a new // ThunkSection placed next to the caller. This allows us to handle the vast // majority of thunks simply, but also handle rare cases where the branch range // is smaller than the target specific spacing. // // The algorithm is expected to create all the thunks that are needed in a // single pass, with a small number of programs needing a second pass due to // the insertion of thunks in the first pass increasing the offset between // callers and callees that were only just in range. // // A consequence of allowing new ThunkSections to be created outside of the // pre-created ThunkSections is that in rare cases calls to Thunks that were in // range in pass K, are out of range in some pass > K due to the insertion of // more Thunks in between the caller and callee. When this happens we retarget // the relocation back to the original target and create another Thunk. // Remove ThunkSections that are empty, this should only be the initial set // precreated on pass 0. // Insert the Thunks for OutputSection OS into their designated place // in the Sections vector, and recalculate the InputSection output section // offsets. // This may invalidate any output section offsets stored outside of InputSection void ThunkCreator::mergeThunks(ArrayRef outputSections) { forEachInputSectionDescription( outputSections, [&](OutputSection *os, InputSectionDescription *isd) { if (isd->thunkSections.empty()) return; // Remove any zero sized precreated Thunks. llvm::erase_if(isd->thunkSections, [](const std::pair &ts) { return ts.first->getSize() == 0; }); // ISD->ThunkSections contains all created ThunkSections, including // those inserted in previous passes. Extract the Thunks created this // pass and order them in ascending outSecOff. std::vector newThunks; for (std::pair ts : isd->thunkSections) if (ts.second == pass) newThunks.push_back(ts.first); llvm::stable_sort(newThunks, [](const ThunkSection *a, const ThunkSection *b) { return a->outSecOff < b->outSecOff; }); // Merge sorted vectors of Thunks and InputSections by outSecOff std::vector tmp; tmp.reserve(isd->sections.size() + newThunks.size()); std::merge(isd->sections.begin(), isd->sections.end(), newThunks.begin(), newThunks.end(), std::back_inserter(tmp), mergeCmp); isd->sections = std::move(tmp); }); } // Find or create a ThunkSection within the InputSectionDescription (ISD) that // is in range of Src. An ISD maps to a range of InputSections described by a // linker script section pattern such as { .text .text.* }. ThunkSection *ThunkCreator::getISDThunkSec(OutputSection *os, InputSection *isec, InputSectionDescription *isd, uint32_t type, uint64_t src) { for (std::pair tp : isd->thunkSections) { ThunkSection *ts = tp.first; uint64_t tsBase = os->addr + ts->outSecOff; uint64_t tsLimit = tsBase + ts->getSize(); if (target->inBranchRange(type, src, (src > tsLimit) ? tsBase : tsLimit)) return ts; } // No suitable ThunkSection exists. This can happen when there is a branch // with lower range than the ThunkSection spacing or when there are too // many Thunks. Create a new ThunkSection as close to the InputSection as // possible. Error if InputSection is so large we cannot place ThunkSection // anywhere in Range. uint64_t thunkSecOff = isec->outSecOff; if (!target->inBranchRange(type, src, os->addr + thunkSecOff)) { thunkSecOff = isec->outSecOff + isec->getSize(); if (!target->inBranchRange(type, src, os->addr + thunkSecOff)) fatal("InputSection too large for range extension thunk " + isec->getObjMsg(src - (os->addr + isec->outSecOff))); } return addThunkSection(os, isd, thunkSecOff); } // Add a Thunk that needs to be placed in a ThunkSection that immediately // precedes its Target. ThunkSection *ThunkCreator::getISThunkSec(InputSection *isec) { ThunkSection *ts = thunkedSections.lookup(isec); if (ts) return ts; // Find InputSectionRange within Target Output Section (TOS) that the // InputSection (IS) that we need to precede is in. OutputSection *tos = isec->getParent(); for (BaseCommand *bc : tos->sectionCommands) { auto *isd = dyn_cast(bc); if (!isd || isd->sections.empty()) continue; InputSection *first = isd->sections.front(); InputSection *last = isd->sections.back(); if (isec->outSecOff < first->outSecOff || last->outSecOff < isec->outSecOff) continue; ts = addThunkSection(tos, isd, isec->outSecOff); thunkedSections[isec] = ts; return ts; } return nullptr; } // Create one or more ThunkSections per OS that can be used to place Thunks. // We attempt to place the ThunkSections using the following desirable // properties: // - Within range of the maximum number of callers // - Minimise the number of ThunkSections // // We follow a simple but conservative heuristic to place ThunkSections at // offsets that are multiples of a Target specific branch range. // For an InputSectionDescription that is smaller than the range, a single // ThunkSection at the end of the range will do. // // For an InputSectionDescription that is more than twice the size of the range, // we place the last ThunkSection at range bytes from the end of the // InputSectionDescription in order to increase the likelihood that the // distance from a thunk to its target will be sufficiently small to // allow for the creation of a short thunk. void ThunkCreator::createInitialThunkSections( ArrayRef outputSections) { uint32_t thunkSectionSpacing = target->getThunkSectionSpacing(); forEachInputSectionDescription( outputSections, [&](OutputSection *os, InputSectionDescription *isd) { if (isd->sections.empty()) return; uint32_t isdBegin = isd->sections.front()->outSecOff; uint32_t isdEnd = isd->sections.back()->outSecOff + isd->sections.back()->getSize(); uint32_t lastThunkLowerBound = -1; if (isdEnd - isdBegin > thunkSectionSpacing * 2) lastThunkLowerBound = isdEnd - thunkSectionSpacing; uint32_t isecLimit; uint32_t prevIsecLimit = isdBegin; uint32_t thunkUpperBound = isdBegin + thunkSectionSpacing; for (const InputSection *isec : isd->sections) { isecLimit = isec->outSecOff + isec->getSize(); if (isecLimit > thunkUpperBound) { addThunkSection(os, isd, prevIsecLimit); thunkUpperBound = prevIsecLimit + thunkSectionSpacing; } if (isecLimit > lastThunkLowerBound) break; prevIsecLimit = isecLimit; } addThunkSection(os, isd, isecLimit); }); } ThunkSection *ThunkCreator::addThunkSection(OutputSection *os, InputSectionDescription *isd, uint64_t off) { auto *ts = make(os, off); ts->partition = os->partition; if ((config->fixCortexA53Errata843419 || config->fixCortexA8) && !isd->sections.empty()) { // The errata fixes are sensitive to addresses modulo 4 KiB. When we add // thunks we disturb the base addresses of sections placed after the thunks // this makes patches we have generated redundant, and may cause us to // generate more patches as different instructions are now in sensitive // locations. When we generate more patches we may force more branches to // go out of range, causing more thunks to be generated. In pathological // cases this can cause the address dependent content pass not to converge. // We fix this by rounding up the size of the ThunkSection to 4KiB, this // limits the insertion of a ThunkSection on the addresses modulo 4 KiB, // which means that adding Thunks to the section does not invalidate // errata patches for following code. // Rounding up the size to 4KiB has consequences for code-size and can // trip up linker script defined assertions. For example the linux kernel // has an assertion that what LLD represents as an InputSectionDescription // does not exceed 4 KiB even if the overall OutputSection is > 128 Mib. // We use the heuristic of rounding up the size when both of the following // conditions are true: // 1.) The OutputSection is larger than the ThunkSectionSpacing. This // accounts for the case where no single InputSectionDescription is // larger than the OutputSection size. This is conservative but simple. // 2.) The InputSectionDescription is larger than 4 KiB. This will prevent // any assertion failures that an InputSectionDescription is < 4 KiB // in size. uint64_t isdSize = isd->sections.back()->outSecOff + isd->sections.back()->getSize() - isd->sections.front()->outSecOff; if (os->size > target->getThunkSectionSpacing() && isdSize > 4096) ts->roundUpSizeForErrata = true; } isd->thunkSections.push_back({ts, pass}); return ts; } static bool isThunkSectionCompatible(InputSection *source, SectionBase *target) { // We can't reuse thunks in different loadable partitions because they might // not be loaded. But partition 1 (the main partition) will always be loaded. if (source->partition != target->partition) return target->partition == 1; return true; } static int64_t getPCBias(RelType type) { if (config->emachine != EM_ARM) return 0; switch (type) { case R_ARM_THM_JUMP19: case R_ARM_THM_JUMP24: case R_ARM_THM_CALL: return 4; default: return 8; } } std::pair ThunkCreator::getThunk(InputSection *isec, Relocation &rel, uint64_t src) { std::vector *thunkVec = nullptr; int64_t addend = rel.addend + getPCBias(rel.type); // We use a ((section, offset), addend) pair to find the thunk position if // possible so that we create only one thunk for aliased symbols or ICFed // sections. There may be multiple relocations sharing the same (section, // offset + addend) pair. We may revert the relocation back to its original // non-Thunk target, so we cannot fold offset + addend. if (auto *d = dyn_cast(rel.sym)) if (!d->isInPlt() && d->section) thunkVec = &thunkedSymbolsBySectionAndAddend[{ {d->section->repl, d->value}, addend}]; if (!thunkVec) thunkVec = &thunkedSymbols[{rel.sym, addend}]; // Check existing Thunks for Sym to see if they can be reused for (Thunk *t : *thunkVec) if (isThunkSectionCompatible(isec, t->getThunkTargetSym()->section) && t->isCompatibleWith(*isec, rel) && target->inBranchRange(rel.type, src, t->getThunkTargetSym()->getVA(rel.addend) + getPCBias(rel.type))) return std::make_pair(t, false); // No existing compatible Thunk in range, create a new one Thunk *t = addThunk(*isec, rel); thunkVec->push_back(t); return std::make_pair(t, true); } // Return true if the relocation target is an in range Thunk. // Return false if the relocation is not to a Thunk. If the relocation target // was originally to a Thunk, but is no longer in range we revert the // relocation back to its original non-Thunk target. bool ThunkCreator::normalizeExistingThunk(Relocation &rel, uint64_t src) { if (Thunk *t = thunks.lookup(rel.sym)) { if (target->inBranchRange(rel.type, src, rel.sym->getVA(rel.addend) + getPCBias(rel.type))) return true; rel.sym = &t->destination; rel.addend = t->addend; if (rel.sym->isInPlt()) rel.expr = toPlt(rel.expr); } return false; } // Process all relocations from the InputSections that have been assigned // to InputSectionDescriptions and redirect through Thunks if needed. The // function should be called iteratively until it returns false. // // PreConditions: // All InputSections that may need a Thunk are reachable from // OutputSectionCommands. // // All OutputSections have an address and all InputSections have an offset // within the OutputSection. // // The offsets between caller (relocation place) and callee // (relocation target) will not be modified outside of createThunks(). // // PostConditions: // If return value is true then ThunkSections have been inserted into // OutputSections. All relocations that needed a Thunk based on the information // available to createThunks() on entry have been redirected to a Thunk. Note // that adding Thunks changes offsets between caller and callee so more Thunks // may be required. // // If return value is false then no more Thunks are needed, and createThunks has // made no changes. If the target requires range extension thunks, currently // ARM, then any future change in offset between caller and callee risks a // relocation out of range error. bool ThunkCreator::createThunks(ArrayRef outputSections) { bool addressesChanged = false; if (pass == 0 && target->getThunkSectionSpacing()) createInitialThunkSections(outputSections); // Create all the Thunks and insert them into synthetic ThunkSections. The // ThunkSections are later inserted back into InputSectionDescriptions. // We separate the creation of ThunkSections from the insertion of the // ThunkSections as ThunkSections are not always inserted into the same // InputSectionDescription as the caller. forEachInputSectionDescription( outputSections, [&](OutputSection *os, InputSectionDescription *isd) { for (InputSection *isec : isd->sections) for (Relocation &rel : isec->relocations) { uint64_t src = isec->getVA(rel.offset); // If we are a relocation to an existing Thunk, check if it is // still in range. If not then Rel will be altered to point to its // original target so another Thunk can be generated. if (pass > 0 && normalizeExistingThunk(rel, src)) continue; if (!target->needsThunk(rel.expr, rel.type, isec->file, src, *rel.sym, rel.addend)) continue; Thunk *t; bool isNew; std::tie(t, isNew) = getThunk(isec, rel, src); if (isNew) { // Find or create a ThunkSection for the new Thunk ThunkSection *ts; if (auto *tis = t->getTargetInputSection()) ts = getISThunkSec(tis); else ts = getISDThunkSec(os, isec, isd, rel.type, src); ts->addThunk(t); thunks[t->getThunkTargetSym()] = t; } // Redirect relocation to Thunk, we never go via the PLT to a Thunk rel.sym = t->getThunkTargetSym(); rel.expr = fromPlt(rel.expr); // On AArch64 and PPC, a jump/call relocation may be encoded as // STT_SECTION + non-zero addend, clear the addend after // redirection. if (config->emachine != EM_MIPS) rel.addend = -getPCBias(rel.type); } for (auto &p : isd->thunkSections) addressesChanged |= p.first->assignOffsets(); }); for (auto &p : thunkedSections) addressesChanged |= p.second->assignOffsets(); // Merge all created synthetic ThunkSections back into OutputSection mergeThunks(outputSections); ++pass; return addressesChanged; } // The following aid in the conversion of call x@GDPLT to call __tls_get_addr // hexagonNeedsTLSSymbol scans for relocations would require a call to // __tls_get_addr. // hexagonTLSSymbolUpdate rebinds the relocation to __tls_get_addr. bool elf::hexagonNeedsTLSSymbol(ArrayRef outputSections) { bool needTlsSymbol = false; forEachInputSectionDescription( outputSections, [&](OutputSection *os, InputSectionDescription *isd) { for (InputSection *isec : isd->sections) for (Relocation &rel : isec->relocations) if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) { needTlsSymbol = true; return; } }); return needTlsSymbol; } void elf::hexagonTLSSymbolUpdate(ArrayRef outputSections) { Symbol *sym = symtab->find("__tls_get_addr"); if (!sym) return; bool needEntry = true; forEachInputSectionDescription( outputSections, [&](OutputSection *os, InputSectionDescription *isd) { for (InputSection *isec : isd->sections) for (Relocation &rel : isec->relocations) if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) { if (needEntry) { addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel, *sym); needEntry = false; } rel.sym = sym; } }); } template void elf::scanRelocations(InputSectionBase &); template void elf::scanRelocations(InputSectionBase &); template void elf::scanRelocations(InputSectionBase &); template void elf::scanRelocations(InputSectionBase &); template void elf::reportUndefinedSymbols(); template void elf::reportUndefinedSymbols(); template void elf::reportUndefinedSymbols(); template void elf::reportUndefinedSymbols(); diff --git a/lld/test/ELF/ppc64-tls-pcrel-gd.s b/lld/test/ELF/ppc64-tls-pcrel-gd.s deleted file mode 100644 index 2220d91fe798..000000000000 --- a/lld/test/ELF/ppc64-tls-pcrel-gd.s +++ /dev/null @@ -1,94 +0,0 @@ -# REQUIRES: ppc -# RUN: split-file %s %t - -# RUN: llvm-mc -filetype=obj -triple=powerpc64le %t/asm -o %t.o -# RUN: llvm-mc -filetype=obj -triple=powerpc64le %t/defs -o %t-defs.o -# RUN: ld.lld --shared %t-defs.o -o %t-defs.so -# RUN: ld.lld -T %t/lds --shared %t.o -o %t-gd.so -# RUN: ld.lld -T %t/lds %t.o %t-defs.so -o %t-gdtoie -# RUN: ld.lld -T %t/lds %t.o %t-defs.o -o %t-gdtole - -# RUN: llvm-readelf -r %t-gd.so | FileCheck %s --check-prefix=GD-RELOC -# RUN: llvm-readelf -s %t-gd.so | FileCheck %s --check-prefix=GD-SYM -# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t-gd.so | FileCheck %s --check-prefix=GD - -# RUN: llvm-readelf -r %t-gdtoie | FileCheck %s --check-prefix=GDTOIE-RELOC -# RUN: llvm-readelf -s %t-gdtoie | FileCheck %s --check-prefix=GDTOIE-SYM -# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t-gdtoie | FileCheck %s --check-prefix=GDTOIE - -# RUN: llvm-readelf -r %t-gdtole | FileCheck %s --check-prefix=GDTOLE-RELOC -# RUN: llvm-readelf -s %t-gdtole | FileCheck %s --check-prefix=GDTOLE-SYM -# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t-gdtole | FileCheck %s --check-prefix=GDTOLE - -## This test checks the General Dynamic PC Relative TLS implementation for lld. -## GD - General Dynamic with no relaxation possible -## GDTOIE - General Dynamic relaxed to Initial Exec -## GDTOLE - General Dynamic relaxed to Local Exec - -#--- lds -SECTIONS { - .text_addr 0x1001000 : { *(.text_addr) } -} - -#--- defs -.section .tbss,"awT",@nobits -.globl x -x: - .long 0 -.globl y -y: - .long 0 - -#--- asm - -# GD-RELOC: Relocation section '.rela.dyn' at offset 0x100b8 contains 4 entries: -# GD-RELOC: 0000000001001160 0000000200000044 R_PPC64_DTPMOD64 0000000000000000 x + 0 -# GD-RELOC: 0000000001001168 000000020000004e R_PPC64_DTPREL64 0000000000000000 x + 0 -# GD-RELOC: 0000000001001170 0000000300000044 R_PPC64_DTPMOD64 0000000000000000 y + 0 -# GD-RELOC: 0000000001001178 000000030000004e R_PPC64_DTPREL64 0000000000000000 y + 0 - -# GD-SYM: Symbol table '.dynsym' contains 4 entries: -# GD-SYM: 2: 0000000000000000 0 TLS GLOBAL DEFAULT UND x -# GD-SYM: 3: 0000000000000000 0 TLS GLOBAL DEFAULT UND y - - -# GDTOIE-RELOC: Relocation section '.rela.dyn' at offset 0x10118 contains 2 entries: -# GDTOIE-RELOC: 00000000010010e0 0000000200000049 R_PPC64_TPREL64 0000000000000000 x + 0 -# GDTOIE-RELOC: 00000000010010e8 0000000300000049 R_PPC64_TPREL64 0000000000000000 y + 0 - -# GDTOIE-SYM: Symbol table '.dynsym' contains 4 entries: -# GDTOIE-SYM: 2: 0000000000000000 0 TLS GLOBAL DEFAULT UND x -# GDTOIE-SYM: 3: 0000000000000000 0 TLS GLOBAL DEFAULT UND y - - -# GDTOLE-RELOC: There are no relocations in this file. - -# GDTOLE-SYM: Symbol table '.symtab' contains 5 entries: -# GDTOLE-SYM: 3: 0000000000000000 0 TLS GLOBAL DEFAULT 3 x -# GDTOLE-SYM: 4: 0000000000000004 0 TLS GLOBAL DEFAULT 3 y - -# GD-LABEL: : -# GD-NEXT: paddi 3, 0, 352, 1 -# GD-NEXT: bl -# GD-NEXT: paddi 3, 0, 356, 1 -# GD-NEXT: bl -# GD-NEXT: blr -# GDTOIE-LABEL: : -# GDTOIE-NEXT: pld 3, 224(0), 1 -# GDTOIE-NEXT: add 3, 3, 13 -# GDTOIE-NEXT: pld 3, 220(0), 1 -# GDTOIE-NEXT: add 3, 3, 13 -# GDTOIE-NEXT: blr -# GDTOLE-LABEL: : -# GDTOLE-NEXT: paddi 3, 13, -28672, 0 -# GDTOLE-NEXT: nop -# GDTOLE-NEXT: paddi 3, 13, -28668, 0 -# GDTOLE-NEXT: nop -# GDTOLE-NEXT: blr -.section .text_addr, "ax", %progbits -GDTwoVal: - paddi 3, 0, x@got@tlsgd@pcrel, 1 - bl __tls_get_addr@notoc(x@tlsgd) - paddi 3, 0, y@got@tlsgd@pcrel, 1 - bl __tls_get_addr@notoc(y@tlsgd) - blr