Index: test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml @@ -0,0 +1,16 @@ +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .debug_foo + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + Content: 416c6c2076697369626c65206f626a656374732c206d616e2c2061726520627574206173207061737465626f617264206d61736b732e + - Name: .notdebug_foo + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + Content: 416c6c2076697369626c65206f626a656374732c206d616e2c2061726520627574206173207061737465626f617264206d61736b732e +... Index: test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test @@ -0,0 +1,5 @@ +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: not llvm-objcopy --compress-debug-sections=zlib-fake %t.o 2>&1 | FileCheck %s + +# CHECK: Invalid or unsupported --compress-debug-sections format: zlib-fake. + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test @@ -0,0 +1,34 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib-gnu %t.o %t-compressed.o +# RUN: wc -c %t.o | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.o | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: All visible obj + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: All visible obj +# CHECK-COMPRESSED: .zdebug_foo: +# CHECK-COMPRESSED: ZLIB + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS: Flags [ +# CHECK-FLAGS-NOT: SHF_COMPRESSED +# CHECK-FLAGS: ] + +# We not always set SHF_COMPRESSED flag regardless of gnu or not. +# CHECK-FLAGS: Name: .zdebug_foo +# CHECK-FLAGS: Flags [ +# CHECK-FLAGS: SHF_COMPRESSED +# CHECK-FLAGS: ] + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 637 +# SIZE-COMPRESSED: 656 + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib.test @@ -0,0 +1,35 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib %t.o %t-compressed.o +# RUN: wc -c %t.o | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.o | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: All visible obj + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: All visible obj +# CHECK-COMPRESSED: .debug_foo: + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS: Flags [ +# CHECK-FLAGS-NEXT: SHF_MERGE +# CHECK-FLAGS-NEXT: SHF_STRINGS +# CHECK-FLAGS-NEXT: ] + +# CHECK-FLAGS: Name: .debug_foo +# CHECK-FLAGS: Flags [ +# CHECK-FLAGS-NEXT: SHF_COMPRESSED +# CHECK-FLAGS-NEXT: SHF_MERGE +# CHECK-FLAGS-NEXT: SHF_STRINGS +# CHECK-FLAGS-NEXT: ] + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 637 +# SIZE-COMPRESSED: 672 + Index: tools/llvm-objcopy/ObjcopyOpts.td =================================================================== --- tools/llvm-objcopy/ObjcopyOpts.td +++ tools/llvm-objcopy/ObjcopyOpts.td @@ -17,6 +17,9 @@ defm output_target : Eq<"output-target">, HelpText<"Format of the output file">, Values<"binary">; +defm compress_debug_sections : Eq<"compress-debug-sections">, + MetaVarName<"[ none | zlib | zlib-gnu ]">, + HelpText<"Enable zlib-gnu or zlib Compression of DWARF debug sections.">; def O : JoinedOrSeparate<["-"], "O">, Alias; defm split_dwo : Eq<"split-dwo">, Index: tools/llvm-objcopy/Object.h =================================================================== --- tools/llvm-objcopy/Object.h +++ tools/llvm-objcopy/Object.h @@ -14,8 +14,12 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/JamCRC.h" #include @@ -39,6 +43,7 @@ class GnuDebugLinkSection; class GroupSection; class SectionIndexSection; +class CompressedSection; class Segment; class Object; struct Symbol; @@ -86,6 +91,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) = 0; virtual void visit(const GroupSection &Sec) = 0; virtual void visit(const SectionIndexSection &Sec) = 0; + virtual void visit(const CompressedSection &Sec) = 0; }; class SectionWriter : public SectionVisitor { @@ -104,6 +110,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) override = 0; virtual void visit(const GroupSection &Sec) override = 0; virtual void visit(const SectionIndexSection &Sec) override = 0; + virtual void visit(const CompressedSection &Sec) override = 0; explicit SectionWriter(Buffer &Buf) : Out(Buf) {} }; @@ -122,6 +129,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit ELFSectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -139,6 +147,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit BinarySectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -246,7 +255,7 @@ class SectionBase { public: - StringRef Name; + std::string Name; Segment *ParentSegment = nullptr; uint64_t HeaderOffset; uint64_t OriginalOffset = std::numeric_limits::max(); @@ -341,7 +350,7 @@ public: OwnedDataSection(StringRef SecName, ArrayRef Data) : Data(std::begin(Data), std::end(Data)) { - Name = SecName; + Name = SecName.str(); Type = ELF::SHT_PROGBITS; Size = Data.size(); OriginalOffset = std::numeric_limits::max(); @@ -350,6 +359,72 @@ void accept(SectionVisitor &Sec) const override; }; +template constexpr const T &max(const T &a, const T &b) { + return (a < b) ? b : a; +} + +LLVM_ATTRIBUTE_NORETURN extern void reportError(StringRef File, Error E); + +class CompressedSection : public SectionBase { + MAKE_SEC_WRITER_FRIEND + + std::vector Data; + SmallVector CompressedData; + DebugCompressionType CompressionType; + +public: + CompressedSection(const SectionBase &Sec, + DebugCompressionType CompressionType) + : Data(std::begin(Sec.OriginalData), std::end(Sec.OriginalData)), + CompressionType(CompressionType) { + + ParentSegment = Sec.ParentSegment; + HeaderOffset = Sec.HeaderOffset; + OriginalOffset = std::numeric_limits::max(); + Index = Sec.Index; + HasSymbol = Sec.HasSymbol; + Addr = Sec.Addr; + Align = Sec.Align; + EntrySize = Sec.EntrySize; + Flags = Sec.Flags; + Info = Sec.Info; + Link = Sec.Link; + NameIndex = Sec.NameIndex; + Offset = Sec.Offset; + Size = Sec.Size; + Type = Sec.Type; + + const bool isGnuStyle = (CompressionType == DebugCompressionType::GNU); + Name = (isGnuStyle ? ".z" : ".") + Sec.Name.substr(1); + OriginalData = ArrayRef(Data.data(), Data.size()); + Type = ELF::SHT_PROGBITS; + + StringRef Contents(reinterpret_cast(OriginalData.data()), + OriginalData.size()); + if (Error E = zlib::compress(Contents, CompressedData)) + objcopy::reportError(Name, std::move(E)); + + Size = CompressedData.size(); + // Increment by Chdr size. + if (isGnuStyle) { + constexpr size_t ChdrSize = sizeof("ZLIB") - 1 + sizeof(uint64_t); + Size += ChdrSize; + } else { + constexpr size_t ChdrSize = + max(max(sizeof(object::Elf_Chdr_Impl), + sizeof(object::Elf_Chdr_Impl)), + max(sizeof(object::Elf_Chdr_Impl), + sizeof(object::Elf_Chdr_Impl))); + Size += ChdrSize; + } + + Flags |= ELF::SHF_COMPRESSED; + } + + void finalize() override; + void accept(SectionVisitor &Visitor) const override; +}; + // There are two types of string tables that can exist, dynamic and not dynamic. // In the dynamic case the string table is allocated. Changing a dynamic string // table would mean altering virtual addresses and thus the memory image. So Index: tools/llvm-objcopy/Object.cpp =================================================================== --- tools/llvm-objcopy/Object.cpp +++ tools/llvm-objcopy/Object.cpp @@ -138,6 +138,39 @@ Visitor.visit(*this); } +void BinarySectionWriter::visit(const CompressedSection &Sec) { + error("Cannot write compressed section '" + Sec.Name + "' "); +} + +template +void ELFSectionWriter::visit(const CompressedSection &Sec) { + uint8_t *Buf = Out.getBufferStart(); + Buf += Sec.Offset; + + const uint64_t DecompressedSize = Sec.OriginalData.size(); + if (Sec.CompressionType == DebugCompressionType::GNU) { + ArrayRef Magic = {'Z', 'L', 'I', 'B'}; + std::copy(Magic.begin(), Magic.end(), Buf); + Buf += Magic.size(); + uint64_t *DecompressedSizePtr = reinterpret_cast(Buf); + *DecompressedSizePtr = support::endian::read64be(&DecompressedSize); + Buf += sizeof(DecompressedSize); + } else { + auto Chdr = reinterpret_cast *>(Buf); + Chdr->ch_type = ELF::ELFCOMPRESS_ZLIB; + Chdr->ch_size = DecompressedSize; + Chdr->ch_addralign = Sec.Align; + Buf += sizeof(*Chdr); + } + + std::copy(Sec.CompressedData.begin(), Sec.CompressedData.end(), Buf); +} + +void CompressedSection::finalize() {} +void CompressedSection::accept(SectionVisitor &Visitor) const { + Visitor.visit(*this); +} + void StringTableSection::addString(StringRef Name) { StrTabBuilder.add(Name); Size = StrTabBuilder.getSize(); Index: tools/llvm-objcopy/llvm-objcopy.cpp =================================================================== --- tools/llvm-objcopy/llvm-objcopy.cpp +++ tools/llvm-objcopy/llvm-objcopy.cpp @@ -175,6 +175,7 @@ bool StripSections = false; bool StripUnneeded = false; bool Weaken = false; + DebugCompressionType CompressionType = DebugCompressionType::None; }; using SectionPred = std::function; @@ -290,12 +291,12 @@ } static bool isDebugSection(const SectionBase &Sec) { - return Sec.Name.startswith(".debug") || Sec.Name.startswith(".zdebug") || - Sec.Name == ".gdb_index"; + return StringRef(Sec.Name).startswith(".debug") || + StringRef(Sec.Name).startswith(".zdebug") || Sec.Name == ".gdb_index"; } static bool isDWOSection(const SectionBase &Sec) { - return Sec.Name.endswith(".dwo"); + return StringRef(Sec.Name).endswith(".dwo"); } static bool onlyKeepDWOPred(const Object &Obj, const SectionBase &Sec) { @@ -405,6 +406,30 @@ object_error::parse_failed); } +static bool isCompressed(const SectionBase &Section) { + return StringRef(Section.Name).startswith(".zdebug") || + (Section.Flags & ELF::SHF_COMPRESSED); +} + +static void appendCompressableSections(const CopyConfig &Config, Object &Obj, + SectionPred &RemovePred) { + std::vector SectionsToRemove; + std::for_each(Obj.sections().begin(), Obj.sections().end(), + [&SectionsToRemove](SectionBase &Section) { + if (!isCompressed(Section) && isDebugSection(Section) && + Section.Name != ".gdb_index") { + SectionsToRemove.push_back(&Section); + } + }); + + for (auto S : SectionsToRemove) { + const SectionBase &Section = *S; + Obj.addSection(Section, Config.CompressionType); + RemovePred = [RemovePred, &Section](const SectionBase &Sec) { + return &Sec == &Section || RemovePred(Sec); + }; + } +} // This function handles the high level operations of GNU objcopy including // handling command line options. It's important to outline certain properties // we expect to hold of the command line operations. Any operation that "keeps" @@ -564,7 +589,7 @@ return true; if (&Sec == Obj.SectionNames) return false; - if (Sec.Name.startswith(".gnu.warning")) + if (StringRef(Sec.Name).startswith(".gnu.warning")) return false; return (Sec.Flags & SHF_ALLOC) == 0; }; @@ -616,6 +641,9 @@ }; } + if (Config.CompressionType != DebugCompressionType::None) + appendCompressableSections(Config, Obj, RemovePred); + Obj.removeSections(RemovePred); if (!Config.SectionsToRename.empty()) { @@ -859,6 +887,19 @@ Config.BinaryArch = getMachineInfo(BinaryArch); } + Config.CompressionType = + StringSwitch( + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)) + .Case("zlib-gnu", DebugCompressionType::GNU) + .Case("zlib", DebugCompressionType::Z) + .Default(DebugCompressionType::None); + + if (Config.CompressionType == DebugCompressionType::None && + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections) != "") { + error("Invalid or unsupported --compress-debug-sections format: " + + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)); + } + Config.SplitDWO = InputArgs.getLastArgValue(OBJCOPY_split_dwo); Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink); Config.SymbolsPrefix = InputArgs.getLastArgValue(OBJCOPY_prefix_symbols);