diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h --- a/lld/MachO/ConcatOutputSection.h +++ b/lld/MachO/ConcatOutputSection.h @@ -40,7 +40,6 @@ void finalize() override; bool needsThunks() const; uint64_t estimateStubsInRangeVA(size_t callIdx) const; - void eraseOmittedInputSections(); void writeTo(uint8_t *buf) const override; diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp --- a/lld/MachO/ConcatOutputSection.cpp +++ b/lld/MachO/ConcatOutputSection.cpp @@ -355,12 +355,3 @@ break; } } - -void ConcatOutputSection::eraseOmittedInputSections() { - // Remove the duplicates from inputs - inputs.erase(std::remove_if(inputs.begin(), inputs.end(), - [](const ConcatInputSection *isec) -> bool { - return isec->shouldOmitFromOutput(); - }), - inputs.end()); -} diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -8,6 +8,7 @@ #include "Driver.h" #include "Config.h" +#include "ICF.h" #include "InputFiles.h" #include "LTO.h" #include "MarkLive.h" @@ -18,6 +19,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" +#include "UnwindInfoSection.h" #include "Writer.h" #include "lld/Common/Args.h" @@ -969,6 +971,44 @@ } } +static void gatherInputSections() { + TimeTraceScope timeScope("Gathering input sections"); + for (const InputFile *file : inputFiles) { + for (const SubsectionMap &map : file->subsections) { + for (const SubsectionEntry &entry : map) { + if (auto isec = dyn_cast(entry.isec)) { + if (isec->isCoalescedWeak()) + continue; + if (isec->segname == segment_names::ld) { + assert(isec->name == section_names::compactUnwind); + in.unwindInfo->addInput(isec); + continue; + } + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(entry.isec)) { + in.cStringSection->addInput(isec); + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSections.size(); + } else if (auto *isec = dyn_cast(entry.isec)) { + in.wordLiteralSection->addInput(isec); + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSections.size(); + } else { + llvm_unreachable("unexpected input section kind"); + } + } + } + } + assert(inputSections.size() < UnspecifiedInputOrder); +} + +static void foldIdenticalLiterals() { + if (in.cStringSection) + in.cStringSection->finalizeContents(); + if (in.wordLiteralSection) + in.wordLiteralSection->finalizeContents(); +} + bool macho::link(ArrayRef argsArr, bool canExitEarly, raw_ostream &stdoutOS, raw_ostream &stderrOS) { lld::stdoutOS = &stdoutOS; @@ -1327,25 +1367,17 @@ inputFiles.insert(make(*buffer, segName, sectName)); } - { - TimeTraceScope timeScope("Gathering input sections"); - // Gather all InputSections into one vector. - for (const InputFile *file : inputFiles) { - for (const SubsectionMap &map : file->subsections) { - for (const SubsectionEntry &entry : map) { - if (auto concatIsec = dyn_cast(entry.isec)) - if (concatIsec->isCoalescedWeak()) - continue; - inputSections.push_back(entry.isec); - } - } - } - assert(inputSections.size() < UnspecifiedInputOrder); - } + gatherInputSections(); if (config->deadStrip) markLive(); + // ICF assumes that all literals have been folded already, so we must run + // foldIdenticalLiterals before foldIdenticalSections. + foldIdenticalLiterals(); + if (config->icfLevel != ICFLevel::none) + foldIdenticalSections(); + // Write to an output file. if (target->wordSize == 8) writeResult(); diff --git a/lld/MachO/ICF.h b/lld/MachO/ICF.h --- a/lld/MachO/ICF.h +++ b/lld/MachO/ICF.h @@ -15,26 +15,7 @@ namespace lld { namespace macho { -class ConcatInputSection; - -class ICF { -public: - ICF(std::vector &inputs); - - void run(); - void segregate(size_t begin, size_t end, - std::function - equals); - size_t findBoundary(size_t begin, size_t end); - void forEachClassRange(size_t begin, size_t end, - std::function func); - void forEachClass(std::function func); - - // ICF needs a copy of the inputs vector because its equivalence-class - // segregation algorithm destroys the proper sequence. - std::vector icfInputs; -}; +void foldIdenticalSections(); } // namespace macho } // namespace lld diff --git a/lld/MachO/ICF.cpp b/lld/MachO/ICF.cpp --- a/lld/MachO/ICF.cpp +++ b/lld/MachO/ICF.cpp @@ -10,7 +10,10 @@ #include "ConcatOutputSection.h" #include "InputSection.h" #include "Symbols.h" +#include "UnwindInfoSection.h" + #include "llvm/Support/Parallel.h" +#include "llvm/Support/TimeProfiler.h" #include @@ -18,6 +21,25 @@ using namespace lld; using namespace lld::macho; +class ICF { +public: + ICF(std::vector &inputs); + + void run(); + void segregate(size_t begin, size_t end, + std::function + equals); + size_t findBoundary(size_t begin, size_t end); + void forEachClassRange(size_t begin, size_t end, + std::function func); + void forEachClass(std::function func); + + // ICF needs a copy of the inputs vector because its equivalence-class + // segregation algorithm destroys the proper sequence. + std::vector icfInputs; +}; + ICF::ICF(std::vector &inputs) { icfInputs.assign(inputs.begin(), inputs.end()); } @@ -276,3 +298,59 @@ begin = mid; } } + +template +DenseSet findFunctionsWithUnwindInfo() { + DenseSet result; + for (ConcatInputSection *isec : in.unwindInfo->getInputs()) { + for (size_t i = 0; i < isec->relocs.size(); ++i) { + Reloc &r = isec->relocs[i]; + assert(target->hasAttr(r.type, RelocAttrBits::UNSIGNED)); + if (r.offset % sizeof(CompactUnwindEntry) != + offsetof(CompactUnwindEntry, functionAddress)) + continue; + result.insert(r.referent.get()); + } + } + return result; +} + +void macho::foldIdenticalSections() { + TimeTraceScope timeScope("Fold Identical Code Sections"); + // The ICF equivalence-class segregation algorithm relies on pre-computed + // hashes of InputSection::data for the ConcatOutputSection::inputs and all + // sections referenced by their relocs. We could recursively traverse the + // relocs to find every referenced InputSection, but that precludes easy + // parallelization. Therefore, we hash every InputSection here where we have + // them all accessible as simple vectors. + std::vector codeSections; + + // ICF can't fold functions with unwind info + DenseSet functionsWithUnwindInfo = + target->wordSize == 8 ? findFunctionsWithUnwindInfo() + : findFunctionsWithUnwindInfo(); + + // If an InputSection is ineligible for ICF, we give it a unique ID to force + // it into an unfoldable singleton equivalence class. Begin the unique-ID + // space at inputSections.size(), so that it will never intersect with + // equivalence-class IDs which begin at 0. Since hashes & unique IDs never + // coexist with equivalence-class IDs, this is not necessary, but might help + // someone keep the numbers straight in case we ever need to debug the + // ICF::segregate() + uint64_t icfUniqueID = inputSections.size(); + for (ConcatInputSection *isec : inputSections) { + bool isHashable = isCodeSection(isec) && !isec->shouldOmitFromOutput() && + !functionsWithUnwindInfo.contains(isec) && + isec->isHashableForICF(); + if (isHashable) { + codeSections.push_back(isec); + } else { + isec->icfEqClass[0] = ++icfUniqueID; + } + } + parallelForEach(codeSections, + [](ConcatInputSection *isec) { isec->hashForICF(); }); + // Now that every input section is either hashed or marked as unique, run the + // segregation algorithm to detect foldable subsections. + ICF(codeSections).run(); +} diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -95,7 +95,7 @@ void markLive(uint64_t off) override { live = true; } bool isCoalescedWeak() const { return wasCoalesced && numRefs == 0; } bool shouldOmitFromOutput() const { return !live || isCoalescedWeak(); } - bool isHashableForICF(bool isText) const; + bool isHashableForICF() const; void hashForICF(); void writeTo(uint8_t *buf); @@ -108,8 +108,6 @@ return isec->kind() == ConcatKind; } - // ICF can't fold functions with LSDA+personality - bool hasPersonality = false; // Points to the surviving section after this one is folded by ICF InputSection *replacement = nullptr; // Equivalence-class ID for ICF @@ -249,7 +247,7 @@ bool isCodeSection(const InputSection *); -extern std::vector inputSections; +extern std::vector inputSections; namespace section_names { diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -24,7 +24,7 @@ using namespace lld; using namespace lld::macho; -std::vector macho::inputSections; +std::vector macho::inputSections; uint64_t InputSection::getFileSize() const { return isZeroFill(flags) ? 0 : getSize(); @@ -47,16 +47,10 @@ // ICF needs to hash any section that might potentially be duplicated so // that it can match on content rather than identity. -bool ConcatInputSection::isHashableForICF(bool isText) const { - if (shouldOmitFromOutput()) - return false; +bool ConcatInputSection::isHashableForICF() const { switch (sectionType(flags)) { case S_REGULAR: - if (isText) - return !hasPersonality; - // One might hope that we could hash __TEXT,__const subsections to fold - // references to duplicated values, but alas, many tests fail. - return false; + return true; case S_CSTRING_LITERALS: case S_4BYTE_LITERALS: case S_8BYTE_LITERALS: diff --git a/lld/MachO/MarkLive.cpp b/lld/MachO/MarkLive.cpp --- a/lld/MachO/MarkLive.cpp +++ b/lld/MachO/MarkLive.cpp @@ -101,10 +101,9 @@ if (auto *stubBinder = dyn_cast_or_null(symtab->find("dyld_stub_binder"))) addSym(stubBinder); - for (InputSection *isec : inputSections) { + for (ConcatInputSection *isec : inputSections) { // Sections marked no_dead_strip if (isec->flags & S_ATTR_NO_DEAD_STRIP) { - assert(isa(isec)); enqueue(isec, 0); continue; } @@ -112,37 +111,33 @@ // mod_init_funcs, mod_term_funcs sections if (sectionType(isec->flags) == S_MOD_INIT_FUNC_POINTERS || sectionType(isec->flags) == S_MOD_TERM_FUNC_POINTERS) { - assert(isa(isec)); enqueue(isec, 0); continue; } + } - // Dead strip runs before UnwindInfoSection handling so we need to keep - // __LD,__compact_unwind alive here. - // But that section contains absolute references to __TEXT,__text and - // keeps most code alive due to that. So we can't just enqueue() the - // section: We must skip the relocations for the functionAddress - // in each CompactUnwindEntry. - // See also scanEhFrameSection() in lld/ELF/MarkLive.cpp. - if (isec->segname == segment_names::ld && - isec->name == section_names::compactUnwind) { - auto concatIsec = cast(isec); - concatIsec->live = true; - const int compactUnwindEntrySize = - target->wordSize == 8 ? sizeof(CompactUnwindEntry) - : sizeof(CompactUnwindEntry); - for (const Reloc &r : isec->relocs) { - // This is the relocation for the address of the function itself. - // Ignore it, else these would keep everything alive. - if (r.offset % compactUnwindEntrySize == 0) - continue; + // Dead strip runs before UnwindInfoSection handling so we need to keep + // __LD,__compact_unwind alive here. + // But that section contains absolute references to __TEXT,__text and + // keeps most code alive due to that. So we can't just enqueue() the + // section: We must skip the relocations for the functionAddress + // in each CompactUnwindEntry. + // See also scanEhFrameSection() in lld/ELF/MarkLive.cpp. + for (ConcatInputSection *isec : in.unwindInfo->getInputs()) { + isec->live = true; + const int compactUnwindEntrySize = + target->wordSize == 8 ? sizeof(CompactUnwindEntry) + : sizeof(CompactUnwindEntry); + for (const Reloc &r : isec->relocs) { + // This is the relocation for the address of the function itself. + // Ignore it, else these would keep everything alive. + if (r.offset % compactUnwindEntrySize == 0) + continue; - if (auto *s = r.referent.dyn_cast()) - addSym(s); - else - enqueue(r.referent.get(), r.addend); - } - continue; + if (auto *s = r.referent.dyn_cast()) + addSym(s); + else + enqueue(r.referent.get(), r.addend); } } @@ -163,13 +158,10 @@ // S_ATTR_LIVE_SUPPORT sections are live if they point _to_ a live section. // Process them in a second pass. - for (InputSection *isec : inputSections) { - if (!isa(isec)) - continue; - auto concatIsec = cast(isec); + for (ConcatInputSection *isec : inputSections) { // FIXME: Check if copying all S_ATTR_LIVE_SUPPORT sections into a // separate vector and only walking that here is faster. - if (!(concatIsec->flags & S_ATTR_LIVE_SUPPORT) || concatIsec->live) + if (!(isec->flags & S_ATTR_LIVE_SUPPORT) || isec->live) continue; for (const Reloc &r : isec->relocs) { diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h --- a/lld/MachO/SyntheticSections.h +++ b/lld/MachO/SyntheticSections.h @@ -546,6 +546,7 @@ WordLiteralSection(); void addInput(WordLiteralInputSection *); + void finalizeContents(); void writeTo(uint8_t *buf) const override; uint64_t getSize() const override { @@ -573,6 +574,8 @@ } private: + std::vector inputs; + template struct Hasher { llvm::hash_code operator()(T v) const { return llvm::hash_value(v); } }; diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -774,7 +774,7 @@ } StabsEntry symStab; - symStab.sect = defined->isec->parent->index; + symStab.sect = defined->isec->canonical()->parent->index; symStab.strx = stringTableSection.addString(defined->getName()); symStab.value = defined->getVA(); @@ -899,7 +899,7 @@ nList->n_value = defined->value; } else { nList->n_type = scope | N_SECT; - nList->n_sect = defined->isec->parent->index; + nList->n_sect = defined->isec->canonical()->parent->index; // For the N_SECT symbol type, n_value is the address of the symbol nList->n_value = defined->getVA(); } @@ -1224,40 +1224,46 @@ void WordLiteralSection::addInput(WordLiteralInputSection *isec) { isec->parent = this; - // We do all processing of the InputSection here, so it will be effectively - // finalized. - isec->isFinal = true; - const uint8_t *buf = isec->data.data(); - switch (sectionType(isec->flags)) { - case S_4BYTE_LITERALS: { - for (size_t off = 0, e = isec->data.size(); off < e; off += 4) { - if (!isec->isLive(off)) - continue; - uint32_t value = *reinterpret_cast(buf + off); - literal4Map.emplace(value, literal4Map.size()); + inputs.push_back(isec); +} + +void WordLiteralSection::finalizeContents() { + for (WordLiteralInputSection *isec : inputs) { + // We do all processing of the InputSection here, so it will be effectively + // finalized. + isec->isFinal = true; + const uint8_t *buf = isec->data.data(); + switch (sectionType(isec->flags)) { + case S_4BYTE_LITERALS: { + for (size_t off = 0, e = isec->data.size(); off < e; off += 4) { + if (!isec->isLive(off)) + continue; + uint32_t value = *reinterpret_cast(buf + off); + literal4Map.emplace(value, literal4Map.size()); + } + break; } - break; - } - case S_8BYTE_LITERALS: { - for (size_t off = 0, e = isec->data.size(); off < e; off += 8) { - if (!isec->isLive(off)) - continue; - uint64_t value = *reinterpret_cast(buf + off); - literal8Map.emplace(value, literal8Map.size()); + case S_8BYTE_LITERALS: { + for (size_t off = 0, e = isec->data.size(); off < e; off += 8) { + if (!isec->isLive(off)) + continue; + uint64_t value = *reinterpret_cast(buf + off); + literal8Map.emplace(value, literal8Map.size()); + } + break; } - break; - } - case S_16BYTE_LITERALS: { - for (size_t off = 0, e = isec->data.size(); off < e; off += 16) { - if (!isec->isLive(off)) - continue; - UInt128 value = *reinterpret_cast(buf + off); - literal16Map.emplace(value, literal16Map.size()); + case S_16BYTE_LITERALS: { + for (size_t off = 0, e = isec->data.size(); off < e; off += 16) { + if (!isec->isLive(off)) + continue; + UInt128 value = *reinterpret_cast(buf + off); + literal16Map.emplace(value, literal16Map.size()); + } + break; + } + default: + llvm_unreachable("invalid literal section type"); } - break; - } - default: - llvm_unreachable("invalid literal section type"); } } diff --git a/lld/MachO/UnwindInfoSection.h b/lld/MachO/UnwindInfoSection.h --- a/lld/MachO/UnwindInfoSection.h +++ b/lld/MachO/UnwindInfoSection.h @@ -27,21 +27,21 @@ class UnwindInfoSection : public SyntheticSection { public: - bool isNeeded() const override { return compactUnwindSection != nullptr; } + bool isNeeded() const override { + return !compactUnwindSection->inputs.empty(); + } uint64_t getSize() const override { return unwindInfoSize; } - virtual void prepareRelocations(ConcatInputSection *) = 0; - - void setCompactUnwindSection(ConcatOutputSection *cuSection) { - compactUnwindSection = cuSection; + virtual void addInput(ConcatInputSection *) = 0; + std::vector getInputs() { + return compactUnwindSection->inputs; } + void prepareRelocations(); protected: - UnwindInfoSection() - : SyntheticSection(segment_names::text, section_names::unwindInfo) { - align = 4; - } + UnwindInfoSection(); + virtual void prepareRelocations(ConcatInputSection *) = 0; - ConcatOutputSection *compactUnwindSection = nullptr; + ConcatOutputSection *compactUnwindSection; uint64_t unwindInfoSize = 0; }; diff --git a/lld/MachO/UnwindInfoSection.cpp b/lld/MachO/UnwindInfoSection.cpp --- a/lld/MachO/UnwindInfoSection.cpp +++ b/lld/MachO/UnwindInfoSection.cpp @@ -103,9 +103,11 @@ EncodingMap localEncodingIndexes; }; -template class UnwindInfoSectionImpl : public UnwindInfoSection { +template +class UnwindInfoSectionImpl final : public UnwindInfoSection { public: void prepareRelocations(ConcatInputSection *) override; + void addInput(ConcatInputSection *) override; void finalize() override; void writeTo(uint8_t *buf) const override; @@ -126,6 +128,25 @@ uint64_t level2PagesOffset = 0; }; +UnwindInfoSection::UnwindInfoSection() + : SyntheticSection(segment_names::text, section_names::unwindInfo) { + align = 4; + compactUnwindSection = + make(section_names::compactUnwind); +} + +void UnwindInfoSection::prepareRelocations() { + for (ConcatInputSection *isec : compactUnwindSection->inputs) + prepareRelocations(isec); +} + +template +void UnwindInfoSectionImpl::addInput(ConcatInputSection *isec) { + assert(isec->segname == segment_names::ld && + isec->name == section_names::compactUnwind); + compactUnwindSection->addInput(isec); +} + // Compact unwind relocations have different semantics, so we handle them in a // separate code path from regular relocations. First, we do not wish to add // rebase opcodes for __LD,__compact_unwind, because that section doesn't @@ -133,8 +154,6 @@ // reside in the GOT and must be treated specially. template void UnwindInfoSectionImpl::prepareRelocations(ConcatInputSection *isec) { - assert(isec->segname == segment_names::ld && - isec->name == section_names::compactUnwind); assert(!isec->shouldOmitFromOutput() && "__compact_unwind section should not be omitted"); @@ -150,13 +169,6 @@ offsetof(CompactUnwindEntry, personality)) continue; - Reloc &rFunc = isec->relocs[++i]; - assert(r.offset == - rFunc.offset + offsetof(CompactUnwindEntry, personality)); - auto *referentIsec = - cast(rFunc.referent.get()); - referentIsec->hasPersonality = true; - if (auto *s = r.referent.dyn_cast()) { if (auto *undefined = dyn_cast(s)) { treatUndefinedSymbol(*undefined); diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp --- a/lld/MachO/Writer.cpp +++ b/lld/MachO/Writer.cpp @@ -9,7 +9,6 @@ #include "Writer.h" #include "ConcatOutputSection.h" #include "Config.h" -#include "ICF.h" #include "InputFiles.h" #include "InputSection.h" #include "MapFile.h" @@ -52,8 +51,6 @@ void scanSymbols(); template void createOutputSections(); template void createLoadCommands(); - void foldIdenticalLiterals(); - void foldIdenticalSections(); void finalizeAddresses(); void finalizeLinkEditSegment(); void assignAddresses(OutputSegment *); @@ -589,18 +586,9 @@ void Writer::scanRelocations() { TimeTraceScope timeScope("Scan relocations"); - for (InputSection *isec : inputSections) { - if (!isa(isec)) + for (ConcatInputSection *isec : inputSections) { + if (isec->shouldOmitFromOutput()) continue; - auto concatIsec = cast(isec); - - if (concatIsec->shouldOmitFromOutput()) - continue; - - if (concatIsec->segname == segment_names::ld) { - in.unwindInfo->prepareRelocations(concatIsec); - continue; - } for (auto it = isec->relocs.begin(); it != isec->relocs.end(); ++it) { Reloc &r = *it; @@ -618,12 +606,18 @@ if (!isa(sym) && validateSymbolRelocation(sym, isec, r)) prepareSymbolRelocation(sym, isec, r); } else { - assert(r.referent.is()); + // Canonicalize the referent so that later accesses in Writer won't + // have to worry about it. Perhaps we should do this for Defined::isec + // too... + auto referentIsec = r.referent.get(); + r.referent = referentIsec->canonical(); if (!r.pcrel) in.rebase->addEntry(isec, r.offset); } } } + + in.unwindInfo->prepareRelocations(); } void Writer::scanSymbols() { @@ -890,26 +884,14 @@ // Then add input sections to output sections. for (const auto &p : enumerate(inputSections)) { - InputSection *isec = p.value(); - OutputSection *osec; - if (auto *concatIsec = dyn_cast(isec)) { - if (concatIsec->shouldOmitFromOutput()) - continue; - NamePair names = maybeRenameSection({isec->segname, isec->name}); - ConcatOutputSection *&concatOsec = concatOutputSections[names]; - if (concatOsec == nullptr) - concatOsec = make(names.second); - concatOsec->addInput(concatIsec); - osec = concatOsec; - } else if (auto *cStringIsec = dyn_cast(isec)) { - in.cStringSection->addInput(cStringIsec); - osec = in.cStringSection; - } else if (auto *litIsec = dyn_cast(isec)) { - in.wordLiteralSection->addInput(litIsec); - osec = in.wordLiteralSection; - } else { - llvm_unreachable("unhandled InputSection type"); - } + ConcatInputSection *isec = p.value(); + if (isec->shouldOmitFromOutput()) + continue; + NamePair names = maybeRenameSection({isec->segname, isec->name}); + ConcatOutputSection *&osec = concatOutputSections[names]; + if (osec == nullptr) + osec = make(names.second); + osec->addInput(isec); osec->inputOrder = std::min(osec->inputOrder, static_cast(p.index())); } @@ -918,12 +900,8 @@ for (const auto &it : concatOutputSections) { StringRef segname = it.first.first; ConcatOutputSection *osec = it.second; - if (segname == segment_names::ld) { - assert(osec->name == section_names::compactUnwind); - in.unwindInfo->setCompactUnwindSection(osec); - } else { - getOrCreateOutputSegment(segname)->addOutputSection(osec); - } + assert(segname != segment_names::ld); + getOrCreateOutputSegment(segname)->addOutputSection(osec); } for (SyntheticSection *ssec : syntheticSections) { @@ -943,57 +921,6 @@ linkEditSegment = getOrCreateOutputSegment(segment_names::linkEdit); } -void Writer::foldIdenticalLiterals() { - if (in.cStringSection) - in.cStringSection->finalizeContents(); - // TODO: WordLiteralSection & CFStringSection should be finalized here too -} - -void Writer::foldIdenticalSections() { - if (config->icfLevel == ICFLevel::none) - return; - ConcatOutputSection *textOutputSection = concatOutputSections.lookup( - maybeRenameSection({segment_names::text, section_names::text})); - if (textOutputSection == nullptr) - return; - - TimeTraceScope timeScope("Fold Identical Code Sections"); - // The ICF equivalence-class segregation algorithm relies on pre-computed - // hashes of InputSection::data for the ConcatOutputSection::inputs and all - // sections referenced by their relocs. We could recursively traverse the - // relocs to find every referenced InputSection, but that precludes easy - // parallelization. Therefore, we hash every InputSection here where we have - // them all accessible as a simple vector. - std::vector hashable; - // If an InputSection is ineligible for ICF, we give it a unique ID to force - // it into an unfoldable singleton equivalence class. Begin the unique-ID - // space at inputSections.size(), so that it will never intersect with - // equivalence-class IDs which begin at 0. Since hashes & unique IDs never - // coexist with equivalence-class IDs, this is not necessary, but might help - // someone keep the numbers straight in case we ever need to debug the - // ICF::segregate() - uint64_t icfUniqueID = inputSections.size(); - for (InputSection *isec : inputSections) { - if (auto *concatIsec = dyn_cast(isec)) { - if (concatIsec->isHashableForICF(isec->parent == textOutputSection)) - hashable.push_back(concatIsec); - else - concatIsec->icfEqClass[0] = ++icfUniqueID; - } - } - // FIXME: hash literal sections here too? - parallelForEach(hashable, - [](ConcatInputSection *isec) { isec->hashForICF(); }); - // Now that every input section is either hashed or marked as unique, - // run the segregation algorithm to detect foldable subsections - ICF(textOutputSection->inputs).run(); - size_t oldSize = textOutputSection->inputs.size(); - textOutputSection->eraseOmittedInputSections(); - size_t newSize = textOutputSection->inputs.size(); - log("ICF kept " + Twine(newSize) + " removed " + Twine(oldSize - newSize) + - " of " + Twine(oldSize)); -} - void Writer::finalizeAddresses() { TimeTraceScope timeScope("Finalize addresses"); uint64_t pageSize = target->getPageSize(); @@ -1125,10 +1052,6 @@ in.stubHelper->setup(); scanSymbols(); createOutputSections(); - // ICF assumes that all literals have been folded already, so we must run - // foldIdenticalLiterals before foldIdenticalSections. - foldIdenticalLiterals(); - foldIdenticalSections(); // After this point, we create no new segments; HOWEVER, we might // yet create branch-range extension thunks for architectures whose // hardware call instructions have limited range, e.g., ARM(64).