Index: lld/ELF/CMakeLists.txt =================================================================== --- lld/ELF/CMakeLists.txt +++ lld/ELF/CMakeLists.txt @@ -23,6 +23,7 @@ Driver.cpp DriverUtils.cpp EhFrame.cpp + EmbedFile.cpp Filesystem.cpp GdbIndex.cpp ICF.cpp Index: lld/ELF/Config.h =================================================================== --- lld/ELF/Config.h +++ lld/ELF/Config.h @@ -80,6 +80,8 @@ llvm::StringMap SectionStartMap; llvm::StringRef Chroot; llvm::StringRef DynamicLinker; + llvm::StringRef Embed; + llvm::StringRef EmbedHint; llvm::StringRef Entry; llvm::StringRef Emulation; llvm::StringRef Fini; @@ -94,6 +96,7 @@ llvm::StringRef SoName; llvm::StringRef Sysroot; llvm::StringRef ThinLTOCacheDir; + llvm::StringRef WriteEmbedHint; std::string Rpath; std::vector VersionDefinitions; std::vector AuxiliaryList; @@ -188,6 +191,7 @@ llvm::Optional ImageBase; uint64_t MaxPageSize; uint64_t ZStackSize; + unsigned EmbedReserveBytes; unsigned LTOPartitions; unsigned LTOO; unsigned Optimize; Index: lld/ELF/Driver.cpp =================================================================== --- lld/ELF/Driver.cpp +++ lld/ELF/Driver.cpp @@ -26,6 +26,7 @@ #include "Driver.h" #include "Config.h" #include "Filesystem.h" +#include "EmbedFile.h" #include "ICF.h" #include "InputFiles.h" #include "InputSection.h" @@ -688,6 +689,10 @@ Config->DynamicLinker = getDynamicLinker(Args); Config->EhFrameHdr = Args.hasFlag(OPT_eh_frame_hdr, OPT_no_eh_frame_hdr, false); + Config->Embed = Args.getLastArgValue(OPT_embed); + Config->EmbedHint = Args.getLastArgValue(OPT_embed_hint); + Config->EmbedReserveBytes = + args::getInteger(Args, OPT_embed_reserve_bytes, 0); Config->EmitRelocs = Args.hasArg(OPT_emit_relocs); Config->EnableNewDtags = Args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true); @@ -759,6 +764,7 @@ Config->WarnCommon = Args.hasFlag(OPT_warn_common, OPT_no_warn_common, false); Config->WarnSymbolOrdering = Args.hasFlag(OPT_warn_symbol_ordering, OPT_no_warn_symbol_ordering, true); + Config->WriteEmbedHint = Args.getLastArgValue(OPT_write_embed_hint); Config->ZCombreloc = getZFlag(Args, "combreloc", "nocombreloc", true); Config->ZCopyreloc = getZFlag(Args, "copyreloc", "nocopyreloc", true); Config->ZExecstack = getZFlag(Args, "execstack", "noexecstack", false); @@ -1307,6 +1313,7 @@ splitSections(); markLive(); demoteSymbols(); + findEmbeddedSections(); mergeSections(); if (Config->ICF) doIcf(); Index: lld/ELF/EmbedFile.h =================================================================== --- /dev/null +++ lld/ELF/EmbedFile.h @@ -0,0 +1,14 @@ +#include + +namespace lld { +namespace elf { + +struct PhdrEntry; + +void findEmbeddedSections(); +void updateEmbeddedStringPieces(); +void addPtEmbed(std::vector &Phdrs); +void writeEmbedHintFile(); + +} +} Index: lld/ELF/EmbedFile.cpp =================================================================== --- /dev/null +++ lld/ELF/EmbedFile.cpp @@ -0,0 +1,231 @@ +#include "EmbedFile.h" +#include "Config.h" +#include "InputFiles.h" +#include "InputSection.h" +#include "OutputSections.h" +#include "Symbols.h" +#include "SyntheticSections.h" +#include "lld/Common/Memory.h" +#include "lld/Common/Strings.h" +#include + +using namespace lld; +using namespace lld::elf; +using namespace llvm; +using namespace llvm::ELF; +using namespace llvm::support; + +static InputSection *EmbedFile; +static std::vector ReservedBytes; + +const uint32_t EmbedHintFileMagic = 0x4648454c; +const uint32_t EmbedHintFileVersion = 0; + +struct EmbedHintFileHeader { + ulittle32_t Magic, Version, NumStrings, NumSections; +}; + +struct StringHint { + ulittle32_t Offset; +}; + +struct SectionHint { + ulittle32_t Offset, Size; +}; + +struct PieceToReplace { + MergeInputSection *MIS; + SectionPiece *Piece; + uint64_t EmbedFileOffset; +}; + +static std::vector PiecesToReplace; + +void elf::findEmbeddedSections() { + if (Config->Embed.empty()) + return; + + auto EmbedFileBuf = readFile(Config->Embed); + if (!EmbedFileBuf) + return; + if (Config->EmbedReserveBytes) { + ReservedBytes.resize(Config->EmbedReserveBytes); + InputSection *ReservedBytesSec = make( + nullptr, SHF_ALLOC, SHT_PROGBITS, 1, ReservedBytes, ".rodata"); + ReservedBytesSec->Live = true; + InputSections.push_back(ReservedBytesSec); + } + + EmbedFile = + make(nullptr, SHF_ALLOC, SHT_PROGBITS, 4096, + toArrayRef(EmbedFileBuf->getBuffer()), ".rodata"); + EmbedFile->Live = true; + InputSections.push_back(EmbedFile); + + auto EmbedHintFile = readFile(Config->EmbedHint); + if (!EmbedHintFile) + return; + + auto *Hdr = reinterpret_cast( + EmbedHintFile->getBufferStart()); + if (EmbedHintFile->getBufferSize() < sizeof(EmbedHintFileHeader) || + Hdr->Magic != EmbedHintFileMagic || Hdr->Version != EmbedHintFileVersion) + fatal("--embed-hint: invalid hint file"); + + ArrayRef StringHintArr{ + reinterpret_cast(Hdr + 1), Hdr->NumStrings}; + ArrayRef SectionHintArr{ + reinterpret_cast(StringHintArr.end()), + Hdr->NumSections}; + + DenseMap StringHints; + for (const StringHint &H : StringHintArr) + StringHints[EmbedFileBuf->getBufferStart() + H.Offset] = H.Offset; + + DenseMap> SectionHints; + for (const SectionHint &H : SectionHintArr) + SectionHints[{EmbedFileBuf->getBufferStart() + H.Offset, H.Size}].insert( + H.Offset); + + // Mapping from input sections to offsets into embedded file. + DenseMap SectionMap; + auto HandleInputSection = [&](InputSection *IS) { + if (IS->Type != SHT_PROGBITS || IS->Flags != SHF_ALLOC || + !IS->Name.startswith(".rodata") || IS->NumRelocations != 0) + return; + auto I = SectionHints.find(toStringRef(IS->Data)); + if (I == SectionHints.end() || I->second.empty()) + return; + for (auto OI = I->second.begin(), OE = I->second.end(); OI != OE; ++OI) { + if (*OI % IS->Alignment == 0) { + SectionMap[IS] = *OI; + I->second.erase(OI); + return; + } + } + }; + + auto HandleMergeInputSection = [&](MergeInputSection *MIS) { + if (MIS->Type != SHT_PROGBITS || !(MIS->Flags & SHF_ALLOC) || + !(MIS->Flags & SHF_STRINGS) || !MIS->Name.startswith(".rodata") || + MIS->Entsize != 1) + return; + for (auto &P : MIS->Pieces) { + if (!P.Live) + continue; + auto I = StringHints.find( + reinterpret_cast(MIS->Data.data() + P.InputOff)); + if (I == StringHints.end()) + continue; + P.Live = false; + P.OutputOff = -I->second; + PiecesToReplace.push_back({MIS, &P, I->second}); + } + }; + + for (InputSectionBase *ISB : InputSections) { + if (!ISB->Live) + continue; + if (auto *IS = dyn_cast(ISB)) + HandleInputSection(IS); + else if (auto *MS = dyn_cast(ISB)) + HandleMergeInputSection(MS); + } + + for (InputFile *F : ObjectFiles) { + for (Symbol *S : F->getSymbols()) { + auto *D = dyn_cast(S); + if (!D) + continue; + auto *IS = dyn_cast_or_null(D->Section); + if (!IS) + continue; + auto I = SectionMap.find(IS); + if (I == SectionMap.end()) + continue; + D->Value += I->second; + D->Section = EmbedFile; + } + } + + for (auto &P : SectionMap) + P.first->Live = false; +} + +void elf::updateEmbeddedStringPieces() { + for (auto &P : PiecesToReplace) { + uint64_t ParentVA = P.MIS->getParent()->getVA(); + uint64_t StringVA = EmbedFile->getVA() + P.EmbedFileOffset; + P.Piece->OutputOff = StringVA - ParentVA; + } +} + +void elf::writeEmbedHintFile() { + if (Config->WriteEmbedHint.empty()) + return; + + std::vector HintSections; + std::vector HintStrings; + + auto HandleInputSection = [&](InputSection *IS) { + if (IS->Type != SHT_PROGBITS || IS->Flags != SHF_ALLOC || + !IS->Name.startswith(".rodata")) + return; + HintSections.push_back(IS); + }; + + auto HandleMergeInputSection = [&](MergeInputSection *MIS) { + if (MIS->Type != SHT_PROGBITS || !(MIS->Flags & SHF_ALLOC) || + !(MIS->Flags & SHF_STRINGS) || !MIS->Name.startswith(".rodata") || + MIS->Entsize != 1) + return; + for (auto &P : MIS->Pieces) { + if (!P.Live) + continue; + HintStrings.push_back(MIS->getParent()->getParent()->Offset + + MIS->getParent()->OutSecOff + P.OutputOff); + } + }; + + for (InputSectionBase *ISB : InputSections) { + if (!ISB->Live) + continue; + if (auto *MSS = dyn_cast(ISB)) + for (auto *MIS : MSS->Sections) + HandleMergeInputSection(MIS); + else if (auto *IS = dyn_cast(ISB)) + HandleInputSection(IS); + } + + std::vector Data; + Data.push_back(EmbedHintFileMagic); + Data.push_back(EmbedHintFileVersion); + Data.push_back(HintStrings.size()); + Data.push_back(HintSections.size()); + + for (uint32_t Off : HintStrings) + Data.push_back(Off); + + for (InputSection *IS : HintSections) { + Data.push_back(IS->getParent()->Offset + IS->OutSecOff); + Data.push_back(IS->Data.size()); + } + + std::error_code EC; + raw_fd_ostream OS(Config->WriteEmbedHint, EC, sys::fs::F_None); + if (EC) { + error("cannot open " + Config->WriteEmbedHint + ": " + EC.message()); + return; + } + + OS << StringRef{reinterpret_cast(Data.data()), Data.size() * 4}; +} + +void elf::addPtEmbed(std::vector &Phdrs) { + if (!EmbedFile) + return; + + auto *Phdr = make(PT_LLVM_EMBED, PF_R); + Phdr->add(EmbedFile); + Phdrs.push_back(Phdr); +} Index: lld/ELF/InputSection.h =================================================================== --- lld/ELF/InputSection.h +++ lld/ELF/InputSection.h @@ -38,6 +38,7 @@ class SectionBase { public: enum Kind { Regular, EHFrame, Merge, Synthetic, Output }; + enum SyntheticKind { SNone, SBss, SMerge }; Kind kind() const { return (Kind)SectionKind; } @@ -59,7 +60,7 @@ // If GC is disabled, all sections are considered live by default. unsigned Live : 1; - unsigned Bss : 1; + unsigned SyntheticSectionKind : 2; // These corresponds to the fields in Elf_Shdr. uint32_t Alignment; @@ -85,8 +86,8 @@ uint64_t Entsize, uint64_t Alignment, uint32_t Type, uint32_t Info, uint32_t Link) : Name(Name), Repl(this), SectionKind(SectionKind), Live(false), - Bss(false), Alignment(Alignment), Flags(Flags), Entsize(Entsize), - Type(Type), Link(Link), Info(Info) {} + SyntheticSectionKind(SNone), Alignment(Alignment), Flags(Flags), + Entsize(Entsize), Type(Type), Link(Link), Info(Info) {} }; // This corresponds to a section of an input file. Index: lld/ELF/InputSection.cpp =================================================================== --- lld/ELF/InputSection.cpp +++ lld/ELF/InputSection.cpp @@ -459,7 +459,7 @@ OutputSection *OS = Sym.getOutputSection(); if (!OS || !OS->PtLoad || !OS->PtLoad->FirstSec) fatal("SBREL relocation to " + Sym.getName() + " without static base"); - return OS->PtLoad->FirstSec->Addr; + return cast(OS->PtLoad->FirstSec)->Addr; } static uint64_t getRelocTargetVA(RelType Type, int64_t A, uint64_t P, Index: lld/ELF/LinkerScript.cpp =================================================================== --- lld/ELF/LinkerScript.cpp +++ lld/ELF/LinkerScript.cpp @@ -13,6 +13,7 @@ #include "LinkerScript.h" #include "Config.h" +#include "EmbedFile.h" #include "InputSection.h" #include "OutputSections.h" #include "SymbolTable.h" @@ -1050,6 +1051,8 @@ assignOffsets(cast(Base)); } Ctx = nullptr; + + updateEmbeddedStringPieces(); } // Creates program headers as instructed by PHDRS linker script command. Index: lld/ELF/MapFile.cpp =================================================================== --- lld/ELF/MapFile.cpp +++ lld/ELF/MapFile.cpp @@ -59,7 +59,8 @@ for (Symbol *B : File->getSymbols()) if (auto *DR = dyn_cast(B)) if (!DR->isSection() && DR->Section && DR->Section->Live && - (DR->File == File || DR->NeedsPltAddr || DR->Section->Bss)) + (DR->File == File || DR->NeedsPltAddr || + isa(DR->Section))) V.push_back(DR); return V; } Index: lld/ELF/Options.td =================================================================== --- lld/ELF/Options.td +++ lld/ELF/Options.td @@ -111,6 +111,13 @@ "Request creation of .eh_frame_hdr section and PT_GNU_EH_FRAME segment header", "Do not create .eh_frame_hdr section">; +defm embed: Eq<"embed">, HelpText<"Path to file to embed for content reuse">; + +defm embed_hint: Eq<"embed-hint">, HelpText<"Path to content reuse embedding hint file">; + +defm embed_reserve_bytes: Eq<"embed-reserve-bytes">, + HelpText<"Number of bytes to reserve before the start of the embedded file (default 0)">; + def emit_relocs: F<"emit-relocs">, HelpText<"Generate relocations in output">; def enable_new_dtags: F<"enable-new-dtags">, @@ -354,6 +361,9 @@ defm wrap: Eq<"wrap">, HelpText<"Use wrapper functions for symbol">, MetaVarName<"">; +defm write_embed_hint: Eq<"write-embed-hint">, + HelpText<"Write content embedding hints to the specified file">; + def z: JoinedOrSeparate<["-"], "z">, MetaVarName<"