diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -275,7 +275,9 @@ subsecMap.push_back({0, isec}); for (uint64_t off = recordSize; off < data.size(); off += recordSize) { // Copying requires less memory than constructing a fresh InputSection. - auto *copy = make(*isec); + auto copy = make( + segname, name, this, data.slice(0, recordSize), align, flags); + copy->data = data.slice(off, recordSize); subsecMap.push_back({off, copy}); } @@ -616,8 +618,7 @@ } } -template -static bool isUndef(const NList &sym) { +template static bool isUndef(const NList &sym) { return (sym.n_type & N_TYPE) == N_UNDF && sym.n_value == 0; } @@ -718,7 +719,7 @@ } auto *concatIsec = cast(isec); - auto *nextIsec = make(*concatIsec); + auto *nextIsec = make(concatIsec); nextIsec->numRefs = 0; nextIsec->wasCoalesced = false; if (isZeroFill(isec->getFlags())) { diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -19,6 +19,9 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/BinaryFormat/MachO.h" +#include +#include + namespace lld { namespace macho { @@ -48,7 +51,7 @@ // The offset from the beginning of the file. uint64_t getVA(uint64_t off) const; // Whether the data at \p off in this InputSection is live. - virtual bool isLive(uint64_t off) const = 0; + virtual bool isLive(uint64_t off) = 0; virtual void markLive(uint64_t off) = 0; virtual InputSection *canonical() { return this; } @@ -87,11 +90,18 @@ const Shared *const shared; }; +// TODO(BEFORE SUBMIT): only need the lock in any isLive() when markLike() is +// running + // ConcatInputSections are combined into (Concat)OutputSections through simple // concatenation, in contrast with literal sections which may have their // contents merged before output. class ConcatInputSection final : public InputSection { public: + ConcatInputSection(ConcatInputSection *isec) + : ConcatInputSection(isec->shared->segname, isec->shared->name, + isec->shared->file, isec->data, isec->align, + isec->shared->flags) {} ConcatInputSection(StringRef segname, StringRef name, InputFile *file, ArrayRef data, uint32_t align = 1, uint32_t flags = 0) @@ -105,8 +115,11 @@ uint64_t getOffset(uint64_t off) const override { return outSecOff + off; } uint64_t getVA() const { return InputSection::getVA(0); } // ConcatInputSections are entirely live or dead, so the offset is irrelevant. - bool isLive(uint64_t off) const override { return live; } - void markLive(uint64_t off) override { live = true; } + bool isLive(uint64_t off) override { return live; } + void markLive(uint64_t off) override { + const std::lock_guard l(liveNessMutex); + live = true; + } bool isCoalescedWeak() const { return wasCoalesced && numRefs == 0; } bool shouldOmitFromOutput() const { return !live || isCoalescedWeak(); } bool isHashableForICF() const; @@ -133,13 +146,14 @@ // while all copies in other translation units are coalesced into the // first and not copied to the output. bool wasCoalesced = false; - bool live = !config->deadStrip; + mutable bool live = !config->deadStrip; // How many symbols refer to this InputSection. uint32_t numRefs = 0; // This variable has two usages. Initially, it represents the input order. // After assignAddresses is called, it represents the offset from the // beginning of the output section this section was assigned to. uint64_t outSecOff = 0; + std::mutex liveNessMutex; }; // Verify ConcatInputSection's size on 64-bit builds. @@ -164,7 +178,7 @@ struct StringPiece { // Offset from the start of the containing input section. uint32_t inSecOff; - uint32_t live : 1; + mutable uint32_t live : 1; // Only set if deduplicating literals uint32_t hash : 31; // Offset from the start of the containing output section. @@ -193,8 +207,14 @@ : InputSection(CStringLiteralKind, segname, name, file, data, align, flags) {} uint64_t getOffset(uint64_t off) const override; - bool isLive(uint64_t off) const override { return getStringPiece(off).live; } - void markLive(uint64_t off) override { getStringPiece(off).live = true; } + bool isLive(uint64_t off) override { + const std::lock_guard l(liveNessMutex); + return getStringPiece(off).live; + } + void markLive(uint64_t off) override { + const std::lock_guard l(liveNessMutex); + getStringPiece(off).live = true; + } // Find the StringPiece that contains this offset. StringPiece &getStringPiece(uint64_t off); const StringPiece &getStringPiece(uint64_t off) const; @@ -222,6 +242,7 @@ } std::vector pieces; + std::mutex liveNessMutex; }; class WordLiteralInputSection final : public InputSection { @@ -230,10 +251,14 @@ ArrayRef data, uint32_t align, uint32_t flags); uint64_t getOffset(uint64_t off) const override; - bool isLive(uint64_t off) const override { + bool isLive(uint64_t off) override { + const std::lock_guard l(liveNessMutex); return live[off >> power2LiteralSize]; } - void markLive(uint64_t off) override { live[off >> power2LiteralSize] = 1; } + void markLive(uint64_t off) override { + const std::lock_guard l(liveNessMutex); + live[off >> power2LiteralSize] = 1; + } static bool classof(const InputSection *isec) { return isec->kind() == WordLiteralKind; @@ -242,7 +267,8 @@ private: unsigned power2LiteralSize; // The liveness of data[off] is tracked by live[off >> power2LiteralSize]. - llvm::BitVector live; + mutable llvm::BitVector live; + std::mutex liveNessMutex; }; inline uint8_t sectionType(uint32_t flags) { diff --git a/lld/MachO/MarkLive.cpp b/lld/MachO/MarkLive.cpp --- a/lld/MachO/MarkLive.cpp +++ b/lld/MachO/MarkLive.cpp @@ -13,8 +13,12 @@ #include "Symbols.h" #include "UnwindInfoSection.h" #include "mach-o/compact_unwind_encoding.h" +#include "llvm/Support/Parallel.h" #include "llvm/Support/TimeProfiler.h" +#include +#include + namespace lld { namespace macho { @@ -33,6 +37,7 @@ // Literal sections cannot contain references to other sections, so we only // store ConcatInputSections in our worklist. SmallVector worklist; + std::mutex listMutex; auto enqueue = [&](InputSection *isec, uint64_t off) { if (isec->isLive(off)) @@ -40,6 +45,7 @@ isec->markLive(off); if (auto s = dyn_cast(isec)) { assert(!s->isCoalescedWeak()); + const std::lock_guard l(listMutex); worklist.push_back(s); } }; @@ -54,7 +60,10 @@ // Add GC roots. if (config->entry) addSym(config->entry); - for (Symbol *sym : symtab->getSymbols()) { + auto symbols = symtab->getSymbols(); + parallelForEachN(0, symbols.size(), [&](size_t i) { + Symbol *sym = symbols[i]; + TimeTraceScope timeScope("scan for symtab"); if (auto *defined = dyn_cast(sym)) { // -exported_symbol(s_list) if (!config->exportedSymbols.empty() && @@ -65,13 +74,13 @@ assert(!defined->privateExtern && "should have been rejected by driver"); addSym(defined); - continue; + return; } // public symbols explicitly marked .no_dead_strip if (defined->referencedDynamically || defined->noDeadStrip) { addSym(defined); - continue; + return; } // FIXME: When we implement these flags, make symbols from them GC roots: @@ -85,10 +94,10 @@ config->outputType != MH_EXECUTE || config->exportDynamic; if (externsAreRoots && !defined->privateExtern) { addSym(defined); - continue; + return; } } - } + }); // -u symbols for (Symbol *sym : config->explicitUndefineds) if (auto *defined = dyn_cast(sym)) @@ -103,20 +112,23 @@ if (auto *stubBinder = dyn_cast_or_null(symtab->find("dyld_stub_binder"))) addSym(stubBinder); - for (ConcatInputSection *isec : inputSections) { + + parallelForEachN(0, inputSections.size(), [&](size_t i) { + ConcatInputSection *isec = inputSections[i]; + TimeTraceScope timeScope("scan input sections"); // Sections marked no_dead_strip if (isec->getFlags() & S_ATTR_NO_DEAD_STRIP) { enqueue(isec, 0); - continue; + return; } // mod_init_funcs, mod_term_funcs sections if (sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS || sectionType(isec->getFlags()) == S_MOD_TERM_FUNC_POINTERS) { enqueue(isec, 0); - continue; + return; } - } + }); // Dead strip runs before UnwindInfoSection handling so we need to keep // __LD,__compact_unwind alive here. @@ -125,7 +137,10 @@ // section: We must skip the relocations for the functionAddress // in each CompactUnwindEntry. // See also scanEhFrameSection() in lld/ELF/MarkLive.cpp. - for (ConcatInputSection *isec : in.unwindInfo->getInputs()) { + auto unwindInfoInputs = in.unwindInfo->getInputs(); + parallelForEachN(0, unwindInfoInputs.size(), [&](size_t i) { + ConcatInputSection *isec = unwindInfoInputs[i]; + TimeTraceScope timeScope("scan unwind info"); isec->live = true; const int compactUnwindEntrySize = target->wordSize == 8 ? sizeof(CompactUnwindEntry) @@ -141,7 +156,7 @@ else enqueue(r.referent.get(), r.addend); } - } + }); do { // Mark things reachable from GC roots as live.