Index: test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml @@ -0,0 +1,16 @@ +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .debug_foo + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + Content: 416c6c2076697369626c65206f626a656374732c206d616e2c2061726520627574206173207061737465626f617264206d61736b732e + - Name: .notdebug_foo + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + Content: 416c6c2076697369626c65206f626a656374732c206d616e2c2061726520627574206173207061737465626f617264206d61736b732e +... Index: test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test @@ -0,0 +1,5 @@ +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: not llvm-objcopy --compress-debug-sections=zlib-fake %t.o 2>&1 | FileCheck %s + +# CHECK: Invalid or unsupported --compress-debug-sections format: zlib-fake. + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test @@ -0,0 +1,30 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib-gnu %t.o %t-compressed.o +# RUN: wc -c %t.o | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.o | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: All visible obj + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: All visible obj +# CHECK-COMPRESSED: .zdebug_foo: +# CHECK-COMPRESSED: ZLIB + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS: Flags [ (0x30) + +# CHECK-FLAGS: Name: .zdebug_foo +# CHECK-FLAGS-NOT: Flags [ (0x8 +# CHECK-FLAGS-NOT: SHF_COMPRESSED (0x800) + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 637 +# SIZE-COMPRESSED: 640 + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib.test @@ -0,0 +1,32 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib %t.o %t-compressed.o +# RUN: wc -c %t.o | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.o | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: All visible obj + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: All visible obj +# CHECK-COMPRESSED: .debug_foo: + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS: Flags [ (0x30) + +# CHECK-FLAGS: Name: .debug_foo +# CHECK-FLAGS: Flags [ (0x8 +# CHECK-FLAGS-NEXT: SHF_COMPRESSED (0x800) +# CHECK-FLAGS-NEXT: SHF_MERGE (0x10) +# CHECK-FLAGS-NEXT: SHF_STRINGS (0x20) +# CHECK-FLAGS-NEXT: ] + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 637 +# SIZE-COMPRESSED: 640 + Index: tools/llvm-objcopy/ObjcopyOpts.td =================================================================== --- tools/llvm-objcopy/ObjcopyOpts.td +++ tools/llvm-objcopy/ObjcopyOpts.td @@ -17,6 +17,9 @@ defm output_target : Eq<"output-target">, HelpText<"Format of the output file">, Values<"binary">; +defm compress_debug_sections : Eq<"compress-debug-sections">, + MetaVarName<"[ none | zlib | zlib-gnu ]">, + HelpText<"Enable zlib-gnu or zlib Compression of DWARF debug sections.">; def O : JoinedOrSeparate<["-"], "O">, Alias; defm split_dwo : Eq<"split-dwo">, Index: tools/llvm-objcopy/Object.h =================================================================== --- tools/llvm-objcopy/Object.h +++ tools/llvm-objcopy/Object.h @@ -14,8 +14,12 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/JamCRC.h" #include @@ -39,6 +43,7 @@ class GnuDebugLinkSection; class GroupSection; class SectionIndexSection; +class CompressedSection; class Segment; class Object; struct Symbol; @@ -86,6 +91,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) = 0; virtual void visit(const GroupSection &Sec) = 0; virtual void visit(const SectionIndexSection &Sec) = 0; + virtual void visit(const CompressedSection &Sec) = 0; }; class SectionWriter : public SectionVisitor { @@ -104,6 +110,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) override = 0; virtual void visit(const GroupSection &Sec) override = 0; virtual void visit(const SectionIndexSection &Sec) override = 0; + virtual void visit(const CompressedSection &Sec) override = 0; explicit SectionWriter(Buffer &Buf) : Out(Buf) {} }; @@ -122,6 +129,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit ELFSectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -139,6 +147,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit BinarySectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -321,11 +330,16 @@ class Section : public SectionBase { MAKE_SEC_WRITER_FRIEND + // TODO: Contents is present in several classes of the hierarchy. + // This needs to be refactored to avoid duplication. ArrayRef Contents; SectionBase *LinkSection = nullptr; public: - explicit Section(ArrayRef Data) : Contents(Data) {} + explicit Section(ArrayRef Data) : Contents(Data) { + // TODO: Unify usage of OriginalData instead of Contents. + OriginalData = Contents; + } void accept(SectionVisitor &Visitor) const override; void removeSectionReferences(const SectionBase *Sec) override; @@ -336,12 +350,16 @@ class OwnedDataSection : public SectionBase { MAKE_SEC_WRITER_FRIEND +protected: + std::string OwnedName; std::vector Data; public: OwnedDataSection(StringRef SecName, ArrayRef Data) : Data(std::begin(Data), std::end(Data)) { - Name = SecName; + OriginalData = ArrayRef(Data.data(), Data.size()); + OwnedName = SecName.str(); + Name = OwnedName; Type = ELF::SHT_PROGBITS; Size = Data.size(); OriginalOffset = std::numeric_limits::max(); @@ -350,6 +368,28 @@ void accept(SectionVisitor &Sec) const override; }; +class CompressedSection : public OwnedDataSection { + MAKE_SEC_WRITER_FRIEND + + DebugCompressionType CompressionType; + +public: + CompressedSection(DebugCompressionType CompressionType, + ArrayRef Data, const SectionBase &Sec) + : OwnedDataSection(Sec.Name, Data), CompressionType(CompressionType) { + OwnedName = ((CompressionType == DebugCompressionType::GNU) ? ".z" : ".") + + Name.substr(Name.startswith(".debug") ? 1 : 2).str(); + Name = OwnedName; + Align = Sec.Align; + Flags |= Sec.Flags; + if (CompressionType != DebugCompressionType::GNU) + Flags |= ELF::SHF_COMPRESSED; + } + + void finalize() override; + void accept(SectionVisitor &Visitor) const override; +}; + // There are two types of string tables that can exist, dynamic and not dynamic. // In the dynamic case the string table is allocated. Changing a dynamic string // table would mean altering virtual addresses and thus the memory image. So Index: tools/llvm-objcopy/Object.cpp =================================================================== --- tools/llvm-objcopy/Object.cpp +++ tools/llvm-objcopy/Object.cpp @@ -138,6 +138,104 @@ Visitor.visit(*this); } +void BinarySectionWriter::visit(const CompressedSection &Sec) { + error("Cannot write compressed section '" + Sec.Name + "' "); +} + +template void writeReserved(support::endian::Writer &W) {} +template <> void writeReserved(support::endian::Writer &W) { + W.write(static_cast(0)); // ch_reserved field. +} +template <> void writeReserved(support::endian::Writer &W) { + W.write(static_cast(0)); // ch_reserved field. +} + +/// Writes the proper compression header info to W depending on CompressionType. +template +Error produceZLibHeader(support::endian::Writer &W, uint64_t DecompressedSize, + unsigned Align, DebugCompressionType CompressionType) { + if (CompressionType != DebugCompressionType::GNU && + CompressionType != DebugCompressionType::Z) { + return createStringError( + llvm::errc::invalid_argument, + "Invalid DebugCompressionType: only GNU and ZLIB are supported."); + } + + if (CompressionType == DebugCompressionType::GNU) { + StringRef Magic = "ZLIB"; + W.OS << Magic; + support::endian::write(W.OS, DecompressedSize, support::big); + return Error::success(); + } + + using Chdr = Elf_Chdr_Impl; + W.write(static_cast(ELF::ELFCOMPRESS_ZLIB)); + writeReserved(W); + W.write(static_cast(DecompressedSize)); + W.write(static_cast(Align)); + return Error::success(); +} + +/// Returns compressed section content, including header data. +/// +/// @param Contents Section Content to be compressed. +/// @param Align Alignment (used for Elf_Chdr). +/// @param CompressionType DebugCompressionType GNU or Z (used for Elf_Chdr). +/// +/// Return value is a tuple that includes if the zlib library hit any errors, +/// followed by a boolean denoting if the compressed content plus the header +/// length is smaller than before, and lastly followed by the actual compressed +/// section content. +template +Expected> compress(StringRef Contents, uint64_t Align, + DebugCompressionType CompressionType) { + SmallVector CompressedContents; + raw_svector_ostream OS(CompressedContents); + support::endian::Writer W(OS, ELFT::TargetEndianness); + if (Error E = + produceZLibHeader(W, Contents.size(), Align, CompressionType)) + return std::move(E); + + SmallVector CompressedBuffer; + if (Error E = zlib::compress(Contents, CompressedBuffer)) + return std::move(E); + W.OS << StringRef(CompressedBuffer.data(), CompressedBuffer.size()).str(); + return std::vector(CompressedContents.begin(), + CompressedContents.end()); +} + +template Expected> +compress(StringRef Contents, uint64_t Align, + DebugCompressionType CompressionType); +template Expected> +compress(StringRef Contents, uint64_t Align, + DebugCompressionType CompressionType); +template Expected> +compress(StringRef Contents, uint64_t Align, + DebugCompressionType CompressionType); +template Expected> +compress(StringRef Contents, uint64_t Align, + DebugCompressionType CompressionType); + +template +void ELFSectionWriter::visit(const CompressedSection &Sec) { + uint8_t *Buf = Out.getBufferStart(); + Buf += Sec.Offset; + StringRef Contents(reinterpret_cast(Sec.OriginalData.data()), + Sec.OriginalData.size()); + Expected> CompressedContentOrError = + compress(Contents, Sec.Align, Sec.CompressionType); + if (!CompressedContentOrError) + reportError("", CompressedContentOrError.takeError()); + std::vector CompressedContents = *CompressedContentOrError; + std::copy(std::begin(CompressedContents), std::end(CompressedContents), Buf); +} + +void CompressedSection::finalize() {} +void CompressedSection::accept(SectionVisitor &Visitor) const { + Visitor.visit(*this); +} + void StringTableSection::addString(StringRef Name) { StrTabBuilder.add(Name); Size = StrTabBuilder.getSize(); Index: tools/llvm-objcopy/llvm-objcopy.cpp =================================================================== --- tools/llvm-objcopy/llvm-objcopy.cpp +++ tools/llvm-objcopy/llvm-objcopy.cpp @@ -175,6 +175,7 @@ bool StripSections = false; bool StripUnneeded = false; bool Weaken = false; + DebugCompressionType CompressionType = DebugCompressionType::None; }; using SectionPred = std::function; @@ -405,6 +406,37 @@ object_error::parse_failed); } +static bool isCompressed(const SectionBase &Section) { + return Section.Name.startswith(".zdebug") || + (Section.Flags & ELF::SHF_COMPRESSED); +} + +static void appendCompressableSections(const CopyConfig &Config, Object &Obj, + SectionPred &RemovePred) { + std::vector Sections; + std::for_each( + Obj.sections().begin(), Obj.sections().end(), + [&Sections](SectionBase &Section) { Sections.push_back(&Section); }); + + for (auto S : Sections) { + const SectionBase &Section = *S; + if (isCompressed(Section) || !isDebugSection(Section) || + Section.Name == ".gdb_index") + continue; + + const uint8_t *BufPtr = + reinterpret_cast(Section.OriginalData.data()); + size_t BufSize = Section.OriginalData.size(); + Obj.addSection(Config.CompressionType, + ArrayRef(BufPtr, BufSize), + Section); + + // Replace this Section with a compressed version. + RemovePred = [RemovePred, &Section](const SectionBase &Sec) { + return &Sec == &Section || RemovePred(Sec); + }; + } +} // This function handles the high level operations of GNU objcopy including // handling command line options. It's important to outline certain properties // we expect to hold of the command line operations. Any operation that "keeps" @@ -616,6 +648,9 @@ }; } + if (Config.CompressionType != DebugCompressionType::None) + appendCompressableSections(Config, Obj, RemovePred); + Obj.removeSections(RemovePred); if (!Config.SectionsToRename.empty()) { @@ -859,6 +894,19 @@ Config.BinaryArch = getMachineInfo(BinaryArch); } + Config.CompressionType = + StringSwitch( + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)) + .Case("zlib-gnu", DebugCompressionType::GNU) + .Case("zlib", DebugCompressionType::Z) + .Default(DebugCompressionType::None); + + if (Config.CompressionType == DebugCompressionType::None && + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections) != "") { + error("Invalid or unsupported --compress-debug-sections format: " + + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)); + } + Config.SplitDWO = InputArgs.getLastArgValue(OBJCOPY_split_dwo); Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink); Config.SymbolsPrefix = InputArgs.getLastArgValue(OBJCOPY_prefix_symbols);