diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp --- a/lld/ELF/Arch/PPC.cpp +++ b/lld/ELF/Arch/PPC.cpp @@ -192,7 +192,7 @@ void PPC::writeGotPlt(uint8_t *buf, const Symbol &s) const { // Address of the symbol resolver stub in .glink . - write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.pltIndex); + write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.getPltIdx()); } bool PPC::needsThunk(RelExpr expr, RelType type, const InputFile *file, diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp --- a/lld/ELF/Arch/PPC64.cpp +++ b/lld/ELF/Arch/PPC64.cpp @@ -1089,7 +1089,7 @@ void PPC64::writePlt(uint8_t *buf, const Symbol &sym, uint64_t /*pltEntryAddr*/) const { - int32_t offset = pltHeaderSize + sym.pltIndex * pltEntrySize; + int32_t offset = pltHeaderSize + sym.getPltIdx() * pltEntrySize; // bl __glink_PLTresolve write32(buf, 0x48000000 | ((-offset) & 0x03FFFFFc)); } diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp --- a/lld/ELF/Arch/X86.cpp +++ b/lld/ELF/Arch/X86.cpp @@ -220,7 +220,7 @@ void X86::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); if (config->isPic) { const uint8_t inst[] = { 0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx) @@ -502,7 +502,7 @@ void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const { uint64_t va = - in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize; + in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize; write32le(buf, va); } @@ -600,7 +600,7 @@ void RetpolinePic::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); const uint8_t insn[] = { 0x50, // pushl %eax 0x8b, 0x83, 0, 0, 0, 0, // mov foo@GOT(%ebx), %eax @@ -659,7 +659,7 @@ void RetpolineNoPic::writePlt(uint8_t *buf, const Symbol &sym, uint64_t pltEntryAddr) const { - unsigned relOff = in.relaPlt->entsize * sym.pltIndex; + unsigned relOff = in.relaPlt->entsize * sym.getPltIdx(); const uint8_t insn[] = { 0x50, // 0: pushl %eax 0xa1, 0, 0, 0, 0, // 1: mov foo_in_GOT, %eax diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -417,7 +417,7 @@ memcpy(buf, inst, sizeof(inst)); write32le(buf + 2, sym.getGotPltVA() - pltEntryAddr - 6); - write32le(buf + 7, sym.pltIndex); + write32le(buf + 7, sym.getPltIdx()); write32le(buf + 12, in.plt->getVA() - pltEntryAddr - 16); } @@ -995,7 +995,7 @@ void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const { uint64_t va = - in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize; + in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize; write64le(buf, va); } @@ -1107,7 +1107,7 @@ write32le(buf + 3, sym.getGotPltVA() - pltEntryAddr - 7); write32le(buf + 8, -off - 12 + 32); write32le(buf + 13, -off - 17 + 18); - write32le(buf + 18, sym.pltIndex); + write32le(buf + 18, sym.getPltIdx()); write32le(buf + 23, -off - 27); } diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -96,6 +96,7 @@ sharedFiles.clear(); backwardReferences.clear(); whyExtract.clear(); + symAux.clear(); tar = nullptr; in.reset(); diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -302,8 +302,7 @@ sym.replace(Defined{sym.file, sym.getName(), sym.binding, sym.stOther, sym.type, value, size, &sec}); - sym.pltIndex = old.pltIndex; - sym.gotIndex = old.gotIndex; + sym.auxIdx = old.auxIdx; sym.verdefIndex = old.verdefIndex; sym.exportDynamic = true; sym.isUsedInRegularObj = true; @@ -1544,15 +1543,17 @@ // may alter section/value, so create a copy of the symbol to make // section/value fixed. auto *directSym = makeDefined(cast(sym)); + directSym->allocateAux(); addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, *directSym); - sym.pltIndex = directSym->pltIndex; + sym.allocateAux(); + symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; if (sym.hasDirectReloc) { // Change the value to the IPLT and redirect all references to it. auto &d = cast(sym); d.section = in.iplt.get(); - d.value = sym.pltIndex * target->ipltEntrySize; + d.value = d.getPltIdx() * target->ipltEntrySize; d.size = 0; // It's important to set the symbol type here so that dynamic loaders // don't try to call the PLT as if it were an ifunc resolver. @@ -1571,6 +1572,10 @@ auto fn = [](Symbol &sym) { if (handleNonPreemptibleIfunc(sym)) return; + if (!sym.needsDynReloc()) + return; + sym.allocateAux(); + if (sym.needsGot) addGotEntry(sym); if (sym.needsPlt) @@ -1584,9 +1589,10 @@ } else { assert(sym.isFunc() && sym.needsPlt); if (!sym.isDefined()) { - replaceWithDefined( - sym, *in.plt, - target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0); + replaceWithDefined(sym, *in.plt, + target->pltHeaderSize + + target->pltEntrySize * sym.getPltIdx(), + 0); sym.needsCopy = true; if (config->emachine == EM_PPC) { // PPC32 canonical PLT entries are at the beginning of .glink @@ -1653,6 +1659,8 @@ if (sym.needsTlsIe && !sym.needsTlsGdToIe) addTpOffsetGotEntry(sym); }; + + assert(symAux.empty()); for (Symbol *sym : symtab->symbols()) fn(*sym); @@ -2173,6 +2181,7 @@ for (Relocation &rel : isec->relocations) if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) { if (needEntry) { + sym->allocateAux(); addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel, *sym); needEntry = false; diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -56,6 +56,16 @@ const uint32_t size; }; +// Some index properties of a symbol are stored separately in this auxiliary +// struct to decrease sizeof(SymbolUnion) in the majority of cases. +struct SymbolAux { + uint32_t gotIdx = -1; + uint32_t pltIdx = -1; + uint32_t tlsGdIdx = -1; +}; + +extern SmallVector symAux; + // The base class for real symbol classes. class Symbol { public: @@ -79,11 +89,10 @@ mutable uint32_t nameSize; public: + // A symAux index used to access GOT/PLT entry indexes. This is allocated in + // postScanRelocations(). + uint32_t auxIdx = -1; uint32_t dynsymIndex = 0; - uint32_t gotIndex = -1; - uint32_t pltIndex = -1; - - uint32_t globalDynIndex = -1; // This field is a index to the symbol's version definition. uint16_t verdefIndex = -1; @@ -191,8 +200,18 @@ return nameData + nameSize; } - bool isInGot() const { return gotIndex != -1U; } - bool isInPlt() const { return pltIndex != -1U; } + uint32_t getGotIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].gotIdx; + } + uint32_t getPltIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].pltIdx; + } + uint32_t getTlsGdIdx() const { + return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].tlsGdIdx; + } + + bool isInGot() const { return getGotIdx() != uint32_t(-1); } + bool isInPlt() const { return getPltIdx() != uint32_t(-1); } uint64_t getVA(int64_t addend = 0) const; @@ -301,6 +320,15 @@ uint8_t needsTlsIe : 1; uint8_t hasDirectReloc : 1; + bool needsDynReloc() const { + return needsCopy || needsGot || needsPlt || needsTlsDesc || needsTlsGd || + needsTlsGdToIe || needsTlsLd || needsGotDtprel || needsTlsIe; + } + void allocateAux() { + auxIdx = symAux.size(); + symAux.emplace_back(); + } + // The partition whose dynamic symbol table contains this symbol's definition. uint8_t partition = 1; @@ -504,9 +532,9 @@ }; // It is important to keep the size of SymbolUnion small for performance and -// memory usage reasons. 80 bytes is a soft limit based on the size of Defined +// memory usage reasons. 72 bytes is a soft limit based on the size of Defined // on a 64-bit system. -static_assert(sizeof(SymbolUnion) <= 80, "SymbolUnion too large"); +static_assert(sizeof(SymbolUnion) <= 72, "SymbolUnion too large"); template struct AssertSymbol { static_assert(std::is_trivially_destructible(), diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp --- a/lld/ELF/Symbols.cpp +++ b/lld/ELF/Symbols.cpp @@ -66,6 +66,7 @@ elf::backwardReferences; SmallVector, 0> elf::whyExtract; +SmallVector elf::symAux; static uint64_t getSymVA(const Symbol &sym, int64_t addend) { switch (sym.kind()) { @@ -161,7 +162,7 @@ } uint64_t Symbol::getGotOffset() const { - return gotIndex * target->gotEntrySize; + return getGotIdx() * target->gotEntrySize; } uint64_t Symbol::getGotPltVA() const { @@ -172,15 +173,15 @@ uint64_t Symbol::getGotPltOffset() const { if (isInIplt) - return pltIndex * target->gotEntrySize; - return (pltIndex + target->gotPltHeaderEntriesNum) * target->gotEntrySize; + return getPltIdx() * target->gotEntrySize; + return (getPltIdx() + target->gotPltHeaderEntriesNum) * target->gotEntrySize; } uint64_t Symbol::getPltVA() const { uint64_t outVA = isInIplt - ? in.iplt->getVA() + pltIndex * target->ipltEntrySize + ? in.iplt->getVA() + getPltIdx() * target->ipltEntrySize : in.plt->getVA() + in.plt->headerSize + - pltIndex * target->pltEntrySize; + getPltIdx() * target->pltEntrySize; // While linking microMIPS code PLT code are always microMIPS // code. Set the less-significant bit to track that fact. diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -650,14 +650,13 @@ } void GotSection::addEntry(Symbol &sym) { - sym.gotIndex = numEntries; - ++numEntries; + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().gotIdx = numEntries++; } bool GotSection::addDynTlsEntry(Symbol &sym) { - if (sym.globalDynIndex != -1U) - return false; - sym.globalDynIndex = numEntries; + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().tlsGdIdx = numEntries; // Global Dynamic TLS entries take two GOT slots. numEntries += 2; return true; @@ -674,11 +673,11 @@ } uint64_t GotSection::getGlobalDynAddr(const Symbol &b) const { - return this->getVA() + b.globalDynIndex * config->wordsize; + return this->getVA() + b.getTlsGdIdx() * config->wordsize; } uint64_t GotSection::getGlobalDynOffset(const Symbol &b) const { - return b.globalDynIndex * config->wordsize; + return b.getTlsGdIdx() * config->wordsize; } void GotSection::finalizeContents() { @@ -972,12 +971,18 @@ } } - // Update Symbol::gotIndex field to use this + // Update SymbolAux::gotIdx field to use this // value later in the `sortMipsSymbols` function. - for (auto &p : primGot->global) - p.first->gotIndex = p.second; - for (auto &p : primGot->relocs) - p.first->gotIndex = p.second; + for (auto &p : primGot->global) { + if (p.first->auxIdx == uint32_t(-1)) + p.first->allocateAux(); + symAux.back().gotIdx = p.second; + } + for (auto &p : primGot->relocs) { + if (p.first->auxIdx == uint32_t(-1)) + p.first->allocateAux(); + symAux.back().gotIdx = p.second; + } // Create dynamic relocations. for (FileGot &got : gots) { @@ -1145,7 +1150,7 @@ } void GotPltSection::addEntry(Symbol &sym) { - assert(sym.pltIndex == entries.size()); + assert(symAux.back().pltIdx == entries.size()); entries.push_back(&sym); } @@ -1190,7 +1195,7 @@ target->gotEntrySize, getIgotPltName()) {} void IgotPltSection::addEntry(Symbol &sym) { - assert(sym.pltIndex == entries.size()); + assert(symAux.back().pltIdx == entries.size()); entries.push_back(&sym); } @@ -2069,7 +2074,7 @@ // Sort entries related to non-local preemptible symbols by GOT indexes. // All other entries go to the beginning of a dynsym in arbitrary order. if (l.sym->isInGot() && r.sym->isInGot()) - return l.sym->gotIndex < r.sym->gotIndex; + return l.sym->getGotIdx() < r.sym->getGotIdx(); if (!l.sym->isInGot() && !r.sym->isInGot()) return false; return !l.sym->isInGot(); @@ -2546,7 +2551,8 @@ } void PltSection::addEntry(Symbol &sym) { - sym.pltIndex = entries.size(); + assert(sym.auxIdx == symAux.size() - 1); + symAux.back().pltIdx = entries.size(); entries.push_back(&sym); } @@ -2592,7 +2598,7 @@ } void IpltSection::addEntry(Symbol &sym) { - sym.pltIndex = entries.size(); + symAux.back().pltIdx = entries.size(); entries.push_back(&sym); }