diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp --- a/lld/ELF/Arch/AArch64.cpp +++ b/lld/ELF/Arch/AArch64.cpp @@ -856,11 +856,11 @@ }; const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop - // needsCopy indicates a non-ifunc canonical PLT entry whose address may + // NEEDS_COPY indicates a non-ifunc canonical PLT entry whose address may // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its // address may escape if referenced by a direct relocation. The condition is // conservative. - bool hasBti = btiHeader && (sym.needsCopy || sym.isInIplt); + bool hasBti = btiHeader && (sym.hasFlag(NEEDS_COPY) || sym.isInIplt); if (hasBti) { memcpy(buf, btiData, sizeof(btiData)); buf += sizeof(btiData); diff --git a/lld/ELF/MapFile.cpp b/lld/ELF/MapFile.cpp --- a/lld/ELF/MapFile.cpp +++ b/lld/ELF/MapFile.cpp @@ -59,7 +59,7 @@ for (Symbol *b : file->getSymbols()) if (auto *dr = dyn_cast(b)) if (!dr->isSection() && dr->section && dr->section->isLive() && - (dr->file == file || dr->needsCopy || dr->section->bss)) + (dr->file == file || dr->hasFlag(NEEDS_COPY) || dr->section->bss)) v.push_back(dr); return v; } diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -296,8 +296,8 @@ // location. static void replaceWithDefined(Symbol &sym, SectionBase &sec, uint64_t value, uint64_t size) { - Symbol old = sym; - + Symbol old{Symbol::PlaceholderKind, nullptr, StringRef(), 0, 0, 0}; + memcpy(&old, &sym, sizeof(old)); sym.replace(Defined{sym.file, StringRef(), sym.binding, sym.stOther, sym.type, value, size, &sec}); @@ -306,7 +306,8 @@ sym.exportDynamic = true; sym.isUsedInRegularObj = true; // A copy relocated alias may need a GOT entry. - sym.needsGot = old.needsGot; + if (old.hasFlag(NEEDS_GOT)) + sym.setFlag(NEEDS_GOT); } // Reserve space in .bss or .bss.rel.ro for copy relocation. @@ -580,6 +581,7 @@ }; std::vector undefs; +std::mutex relocMutex; } // Check whether the definition name def is a mangled function name that matches @@ -822,6 +824,7 @@ // Returns true if the undefined symbol will produce an error message. static bool maybeReportUndefined(Undefined &sym, InputSectionBase &sec, uint64_t offset) { + std::lock_guard lock(relocMutex); // If versioned, issue an error (even if the symbol is weak) because we don't // know the defining filename which is required to construct a Verneed entry. if (sym.hasVersionSuffix) { @@ -1056,11 +1059,13 @@ if (canWrite) { RelType rel = target.getDynRel(type); if (expr == R_GOT || (rel == target.symbolicRel && !sym.isPreemptible)) { + std::lock_guard lock(relocMutex); addRelativeReloc(sec, offset, sym, addend, expr, type); return; } else if (rel != 0) { if (config->emachine == EM_MIPS && rel == target.symbolicRel) rel = target.relativeRel; + std::lock_guard lock(relocMutex); sec.getPartition().relaDyn->addSymbolReloc(rel, sec, offset, sym, addend, type); @@ -1102,7 +1107,7 @@ " against symbol '" + toString(*ss) + "'; recompile with -fPIC or remove '-z nocopyreloc'" + getLocation(sec, sym, offset)); - sym.needsCopy = true; + sym.setFlag(NEEDS_COPY); } sec.relocations.push_back({expr, type, offset, addend, &sym}); return; @@ -1140,8 +1145,7 @@ errorOrWarn("symbol '" + toString(sym) + "' cannot be preempted; recompile with -fPIE" + getLocation(sec, sym, offset)); - sym.needsCopy = true; - sym.needsPlt = true; + sym.setFlag(NEEDS_COPY | NEEDS_PLT); sec.relocations.push_back({expr, type, offset, addend, &sym}); return; } @@ -1195,7 +1199,7 @@ R_TLSDESC_GOTPLT>(expr) && config->shared) { if (expr != R_TLSDESC_CALL) { - sym.needsTlsDesc = true; + sym.setFlag(NEEDS_TLSDESC); c.relocations.push_back({expr, type, offset, addend, &sym}); } return 1; @@ -1249,7 +1253,7 @@ // Local-Dynamic sequence where offset of tls variable relative to dynamic // thread pointer is stored in the got. This cannot be relaxed to Local-Exec. if (expr == R_TLSLD_GOT_OFF) { - sym.needsGotDtprel = true; + sym.setFlag(NEEDS_GOT_DTPREL); c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } @@ -1257,7 +1261,7 @@ if (oneof(expr)) { if (!toExecRelax) { - sym.needsTlsGd = true; + sym.setFlag(NEEDS_TLSGD); c.relocations.push_back({expr, type, offset, addend, &sym}); return 1; } @@ -1285,7 +1289,7 @@ c.relocations.push_back( {R_RELAX_TLS_IE_TO_LE, type, offset, addend, &sym}); } else if (expr != R_TLSIE_HINT) { - sym.needsTlsIe = true; + sym.setFlag(NEEDS_TLSIE); // R_GOT needs a relative relocation for PIC on i386 and Hexagon. if (expr == R_GOT && config->isPic && !target->usesOnlyLowPageBits(type)) addRelativeReloc(c, offset, sym, addend, expr, type); @@ -1373,10 +1377,10 @@ // The 5 types that relative GOTPLT are all x86 and x86-64 specific. if (oneof(expr)) { - in.gotPlt->hasGotPltOffRel = true; + in.gotPlt->hasGotPltOffRel.store(true, std::memory_order_relaxed); } else if (oneof(expr)) { - in.got->hasGotOffRel = true; + in.got->hasGotOffRel.store(true, std::memory_order_relaxed); } // Process TLS relocations, including relaxing TLS relocations. Note that @@ -1423,6 +1427,7 @@ // We were asked not to generate PLT entries for ifuncs. Instead, pass the // direct relocation on through. if (sym.isGnuIFunc() && config->zIfuncNoplt) { + std::lock_guard lock(relocMutex); sym.exportDynamic = true; mainPart->relaDyn->addSymbolReloc(type, sec, offset, sym, addend, type); return; @@ -1439,12 +1444,12 @@ // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf in.mipsGot->addEntry(*sec.file, sym, addend, expr); } else { - sym.needsGot = true; + sym.setFlag(NEEDS_GOT); } } else if (needsPlt(expr)) { - sym.needsPlt = true; + sym.setFlag(NEEDS_PLT); } else { - sym.hasDirectReloc = true; + sym.setFlag(HAS_DIRECT_RELOC); } processAux(expr, type, offset, sym, addend); @@ -1525,7 +1530,7 @@ scanner.template scan(rels.relas); } -static bool handleNonPreemptibleIfunc(Symbol &sym) { +static bool handleNonPreemptibleIfunc(Symbol &sym, uint8_t flags) { // Handle a reference to a non-preemptible ifunc. These are special in a // few ways: // @@ -1569,7 +1574,7 @@ if (!sym.isGnuIFunc() || sym.isPreemptible || config->zIfuncNoplt) return false; // Skip unreferenced non-preemptible ifunc. - if (!(sym.needsGot || sym.needsPlt || sym.hasDirectReloc)) + if (!(flags & (NEEDS_GOT | NEEDS_PLT | HAS_DIRECT_RELOC))) return true; sym.isInIplt = true; @@ -1585,7 +1590,7 @@ sym.allocateAux(); symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; - if (sym.hasDirectReloc) { + if (flags & HAS_DIRECT_RELOC) { // Change the value to the IPLT and redirect all references to it. auto &d = cast(sym); d.section = in.iplt.get(); @@ -1595,9 +1600,9 @@ // don't try to call the PLT as if it were an ifunc resolver. d.type = STT_FUNC; - if (sym.needsGot) + if (flags & NEEDS_GOT) addGotEntry(sym); - } else if (sym.needsGot) { + } else if (flags & NEEDS_GOT) { // Redirect GOT accesses to point to the Igot. sym.gotInIgot = true; } @@ -1606,30 +1611,31 @@ void elf::postScanRelocations() { auto fn = [](Symbol &sym) { - if (handleNonPreemptibleIfunc(sym)) + auto flags = sym.flags.load(std::memory_order_relaxed); + if (handleNonPreemptibleIfunc(sym, flags)) return; if (!sym.needsDynReloc()) return; sym.allocateAux(); - if (sym.needsGot) + if (flags & NEEDS_GOT) addGotEntry(sym); - if (sym.needsPlt) + if (flags & NEEDS_PLT) addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel, sym); - if (sym.needsCopy) { + if (flags & NEEDS_COPY) { if (sym.isObject()) { invokeELFT(addCopyRelSymbol, cast(sym)); - // needsCopy is cleared for sym and its aliases so that in later - // iterations aliases won't cause redundant copies. - assert(!sym.needsCopy); + // NEEDS_COPY is cleared for sym and its aliases so that in + // later iterations aliases won't cause redundant copies. + assert(!sym.hasFlag(NEEDS_COPY)); } else { - assert(sym.isFunc() && sym.needsPlt); + assert(sym.isFunc() && sym.hasFlag(NEEDS_PLT)); if (!sym.isDefined()) { replaceWithDefined(sym, *in.plt, target->pltHeaderSize + target->pltEntrySize * sym.getPltIdx(), 0); - sym.needsCopy = true; + sym.setFlag(NEEDS_COPY); if (config->emachine == EM_PPC) { // PPC32 canonical PLT entries are at the beginning of .glink cast(sym).value = in.plt->headerSize; @@ -1644,13 +1650,13 @@ return; bool isLocalInExecutable = !sym.isPreemptible && !config->shared; - if (sym.needsTlsDesc) { + if (flags & NEEDS_TLSDESC) { in.got->addTlsDescEntry(sym); mainPart->relaDyn->addAddendOnlyRelocIfNonPreemptible( target->tlsDescRel, *in.got, in.got->getTlsDescOffset(sym), sym, target->tlsDescRel); } - if (sym.needsTlsGd) { + if (flags & NEEDS_TLSGD) { in.got->addDynTlsEntry(sym); uint64_t off = in.got->getGlobalDynOffset(sym); if (isLocalInExecutable) @@ -1676,13 +1682,13 @@ mainPart->relaDyn->addSymbolReloc(target->tlsGotRel, *in.got, sym.getGotOffset(), sym); } - if (sym.needsGotDtprel) { + if (flags & NEEDS_GOT_DTPREL) { in.got->addEntry(sym); in.got->relocations.push_back( {R_ABS, target->tlsOffsetRel, sym.getGotOffset(), 0, &sym}); } - if (sym.needsTlsIe && !sym.needsTlsGdToIe) + if ((flags & NEEDS_TLSIE) && !sym.needsTlsGdToIe) addTpOffsetGotEntry(sym); }; diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -39,6 +39,19 @@ class LazyObject; class InputFile; +enum { + NEEDS_GOT = 1 << 0, + NEEDS_PLT = 1 << 1, + HAS_DIRECT_RELOC = 1 << 2, + // True if this symbol needs a canonical PLT entry, or (during + // postScanRelocations) a copy relocation. + NEEDS_COPY = 1 << 3, + NEEDS_TLSDESC = 1 << 4, + NEEDS_TLSGD = 1 << 5, + NEEDS_GOT_DTPREL = 1 << 6, + NEEDS_TLSIE = 1 << 7, +}; + // Some index properties of a symbol are stored separately in this auxiliary // struct to decrease sizeof(SymbolUnion) in the majority of cases. struct SymbolAux { @@ -240,7 +253,7 @@ inline size_t getSymbolSize() const; -protected: +public: Symbol(Kind k, InputFile *file, StringRef name, uint8_t binding, uint8_t stOther, uint8_t type) : file(file), nameData(name.data()), nameSize(name.size()), type(type), @@ -250,12 +263,8 @@ inDynamicList(false), referenced(false), referencedAfterWrap(false), traced(false), hasVersionSuffix(false), isInIplt(false), gotInIgot(false), folded(false), needsTocRestore(false), - scriptDefined(false), needsCopy(false), needsGot(false), - needsPlt(false), needsTlsDesc(false), needsTlsGd(false), - needsTlsGdToIe(false), needsGotDtprel(false), needsTlsIe(false), - hasDirectReloc(false) {} + scriptDefined(false), needsTlsGdToIe(false) {} -public: // True if this symbol is in the Iplt sub-section of the Plt and the Igot // sub-section of the .got.plt or .got. uint8_t isInIplt : 1; @@ -277,20 +286,11 @@ // of the symbol. uint8_t scriptDefined : 1; - // True if this symbol needs a canonical PLT entry, or (during - // postScanRelocations) a copy relocation. - uint8_t needsCopy : 1; + uint8_t needsTlsGdToIe : 1; // Temporary flags used to communicate which symbol entries need PLT and GOT // entries during postScanRelocations(); - uint8_t needsGot : 1; - uint8_t needsPlt : 1; - uint8_t needsTlsDesc : 1; - uint8_t needsTlsGd : 1; - uint8_t needsTlsGdToIe : 1; - uint8_t needsGotDtprel : 1; - uint8_t needsTlsIe : 1; - uint8_t hasDirectReloc : 1; + std::atomic flags = 0; // A symAux index used to access GOT/PLT entry indexes. This is allocated in // postScanRelocations(). @@ -303,9 +303,16 @@ // Version definition index. uint16_t versionId; + void setFlag(uint8_t bit) { flags.fetch_or(bit, std::memory_order_relaxed); } + bool hasFlag(uint8_t bit) const { + return flags.load(std::memory_order_relaxed) & bit; + } + bool needsDynReloc() const { - return needsCopy || needsGot || needsPlt || needsTlsDesc || needsTlsGd || - needsTlsGdToIe || needsGotDtprel || needsTlsIe; + return flags.load(std::memory_order_relaxed) & + (NEEDS_COPY | NEEDS_GOT | NEEDS_PLT | NEEDS_TLSDESC | + NEEDS_TLSGD | NEEDS_GOT_DTPREL | NEEDS_TLSIE) || + needsTlsGdToIe; } void allocateAux() { assert(auxIdx == uint32_t(-1)); @@ -519,7 +526,8 @@ // it over to "this". This function is called as a result of name // resolution, e.g. to replace an undefind symbol with a defined symbol. void Symbol::replace(const Symbol &other) { - Symbol old = *this; + Symbol old{PlaceholderKind, nullptr, StringRef(), 0, 0, 0}; + memcpy(&old, this, sizeof(old)); memcpy(this, &other, other.getSymbolSize()); // old may be a placeholder. The referenced fields must be initialized in @@ -550,6 +558,13 @@ Defined(std::forward(args)...); } +inline Defined *makeDefined(Defined &o) { + auto *ret = reinterpret_cast( + getSpecificAllocSingleton().Allocate()); + memcpy(ret, &o, sizeof(o)); + return ret; +} + void reportDuplicate(const Symbol &sym, const InputFile *newFile, InputSectionBase *errSec, uint64_t errOffset); void maybeWarnUnorderableSymbol(const Symbol *sym); diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp --- a/lld/ELF/Symbols.cpp +++ b/lld/ELF/Symbols.cpp @@ -122,7 +122,7 @@ // field etc) do the same trick as compiler uses to mark microMIPS // for CPU - set the less-significant bit. if (config->emachine == EM_MIPS && isMicroMips() && - ((sym.stOther & STO_MIPS_MICROMIPS) || sym.needsCopy)) + ((sym.stOther & STO_MIPS_MICROMIPS) || sym.hasFlag(NEEDS_COPY))) va |= 1; if (d.isTls() && !config->relocatable) { diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -115,7 +115,7 @@ // Flag to force GOT to be in output if we have relocations // that relies on its address. - bool hasGotOffRel = false; + std::atomic hasGotOffRel = false; protected: size_t numEntries = 0; @@ -357,7 +357,7 @@ // Flag to force GotPlt to be in output if we have relocations // that relies on its address. - bool hasGotPltOffRel = false; + std::atomic hasGotPltOffRel = false; private: SmallVector entries; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -2175,8 +2175,8 @@ } static uint32_t getSymSectionIndex(Symbol *sym) { - assert(!(sym->needsCopy && sym->isObject())); - if (!isa(sym) || sym->needsCopy) + assert(!(sym->hasFlag(NEEDS_COPY) && sym->isObject())); + if (!isa(sym) || sym->hasFlag(NEEDS_COPY)) return SHN_UNDEF; if (const OutputSection *os = sym->getOutputSection()) return os->sectionIndex >= SHN_LORESERVE ? (uint32_t)SHN_XINDEX @@ -2245,7 +2245,7 @@ for (SymbolTableEntry &ent : symbols) { Symbol *sym = ent.sym; - if (sym->isInPlt() && sym->needsCopy) + if (sym->isInPlt() && sym->hasFlag(NEEDS_COPY)) eSym->st_other |= STO_MIPS_PLT; if (isMicroMips()) { // We already set the less-significant bit for symbols @@ -2256,7 +2256,7 @@ // like `objdump` will be able to deal with a correct // symbol position. if (sym->isDefined() && - ((sym->stOther & STO_MIPS_MICROMIPS) || sym->needsCopy)) { + ((sym->stOther & STO_MIPS_MICROMIPS) || sym->hasFlag(NEEDS_COPY))) { if (!strTabSec.isDynamic()) eSym->st_value &= ~1; eSym->st_other |= STO_MIPS_MICROMIPS; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -1912,15 +1912,34 @@ // determine if it needs special treatment, such as creating GOT, PLT, // copy relocations, etc. Note that relocations for non-alloc sections are // directly processed by InputSection::relocateNonAlloc. - for (InputSectionBase *sec : inputSections) - if (sec->isLive() && (sec->flags & SHF_ALLOC)) - scanRelocations(*sec); - for (Partition &part : partitions) { - for (EhInputSection *sec : part.ehFrame->sections) - scanRelocations(*sec); - if (part.armExidx && part.armExidx->isLive()) - for (InputSection *sec : part.armExidx->exidxSections) + { + // MIPS and PPC64 use global states which are not suitable for + // parallelism. + bool serial = + config->emachine == EM_MIPS || config->emachine == EM_PPC64; + parallel::TaskGroup tg; + for (ELFFileBase *f : ctx->objectFiles) { + auto fn = [f]() { + for (InputSectionBase *s : f->getSections()) { + if (s && s->kind() == SectionBase::Regular && s->isLive() && + (s->flags & SHF_ALLOC) && + !(s->type == SHT_ARM_EXIDX && config->emachine == EM_ARM)) + scanRelocations(*s); + } + }; + if (serial) + fn(); + else + tg.execute(fn); + } + + for (Partition &part : partitions) { + for (EhInputSection *sec : part.ehFrame->sections) scanRelocations(*sec); + if (part.armExidx && part.armExidx->isLive()) + for (InputSection *sec : part.armExidx->exidxSections) + scanRelocations(*sec); + } } reportUndefinedSymbols(); diff --git a/lld/test/ELF/undef-multi.s b/lld/test/ELF/undef-multi.s --- a/lld/test/ELF/undef-multi.s +++ b/lld/test/ELF/undef-multi.s @@ -1,7 +1,7 @@ # REQUIRES: x86 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/undef.s -o %t2.o -# RUN: not ld.lld %t.o %t2.o -o /dev/null 2>&1 | FileCheck %s +# RUN: not ld.lld --threads=1 %t.o %t2.o -o /dev/null 2>&1 | FileCheck %s # CHECK: error: undefined symbol: zed2 # CHECK-NEXT: >>> referenced by undef-multi.s @@ -24,7 +24,7 @@ # RUN: echo " call zed2" >> %t.moreref.s # RUN: echo " call zed2" >> %t.moreref.s # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %t.moreref.s -o %t3.o -# RUN: not ld.lld %t.o %t2.o %t3.o -o /dev/null -error-limit=2 2>&1 | \ +# RUN: not ld.lld --threads=1 %t.o %t2.o %t3.o -o /dev/null -error-limit=2 2>&1 | \ # RUN: FileCheck --check-prefix=LIMIT %s # LIMIT: error: undefined symbol: zed2 diff --git a/lld/test/ELF/undef.s b/lld/test/ELF/undef.s --- a/lld/test/ELF/undef.s +++ b/lld/test/ELF/undef.s @@ -5,9 +5,9 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/undef-bad-debug.s -o %t4.o # RUN: rm -f %t2.a # RUN: llvm-ar rc %t2.a %t2.o -# RUN: not ld.lld %t.o %t2.a %t3.o %t4.o -o /dev/null 2>&1 \ +# RUN: not ld.lld --threads=1 %t.o %t2.a %t3.o %t4.o -o /dev/null 2>&1 \ # RUN: | FileCheck %s --implicit-check-not="error:" --implicit-check-not="warning:" -# RUN: not ld.lld -pie %t.o %t2.a %t3.o %t4.o -o /dev/null 2>&1 \ +# RUN: not ld.lld --threads=1 -pie %t.o %t2.a %t3.o %t4.o -o /dev/null 2>&1 \ # RUN: | FileCheck %s --implicit-check-not="error:" --implicit-check-not="warning:" # CHECK: error: undefined symbol: foo