Index: test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml @@ -0,0 +1,14 @@ +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .debug_foo + Type: SHT_PROGBITS + Content: 666f72747974776f + - Name: .notdebug_foo + Type: SHT_PROGBITS + Content: 666f72747974776f +... Index: test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test @@ -0,0 +1,5 @@ +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: not llvm-objcopy --compress-debug-sections=zlib-fake %t.o 2>&1 | FileCheck %s + +# CHECK: Invalid or unsupported --compress-debug-sections format: zlib-fake. + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test @@ -0,0 +1,37 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib-gnu %t.o %t-compressed.o +# RUN: llvm-objcopy --dump-section .debug_foo=%t.sec %t.o +# RUN: llvm-objcopy --dump-section .zdebug_foo=%t-compressed.sec %t-compressed.o +# RUN: wc -c %t.sec | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.sec | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: fortytwo + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: fortytwo +# CHECK-COMPRESSED: .zdebug_foo: +# CHECK-COMPRESSED: ZLIB + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS +# CHECK-FLAGS-NEXT: Flags [ +# CHECK-FLAGS-NEXT: ] + +# We not always set SHF_COMPRESSED flag regardless of gnu or not. +# CHECK-FLAGS-NOT: Name: .debug_foo +# CHECK-FLAGS: Name: .zdebug_foo +# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS +# CHECK-FLAGS-NEXT: Flags [ +# CHECK-FLAGS-NEXT: ] + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 8 +# SIZE-COMPRESSED: 28 + Index: test/tools/llvm-objcopy/compress-debug-sections-zlib.test =================================================================== --- /dev/null +++ test/tools/llvm-objcopy/compress-debug-sections-zlib.test @@ -0,0 +1,36 @@ +# REQUIRES: zlib + +# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o +# RUN: llvm-objcopy --compress-debug-sections=zlib %t.o %t-compressed.o +# RUN: llvm-objcopy --dump-section .debug_foo=%t.sec %t.o +# RUN: llvm-objcopy --dump-section .debug_foo=%t-compressed.sec %t-compressed.o +# RUN: wc -c %t.sec | FileCheck %s -check-prefix=SIZE-ORIGINAL +# RUN: wc -c %t-compressed.sec | FileCheck %s -check-prefix=SIZE-COMPRESSED + +# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s +# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED +# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS + +# CHECK: .debug_foo: +# CHECK: fortytwo + +# CHECK-COMPRESSED: .notdebug_foo: +# CHECK-COMPRESSED: fortytwo +# CHECK-COMPRESSED: .debug_foo: + +# CHECK-FLAGS: Name: .notdebug_foo +# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS +# CHECK-FLAGS-NEXT: Flags [ +# CHECK-FLAGS-NEXT: ] + +# CHECK-FLAGS: Name: .debug_foo +# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS +# CHECK-FLAGS-NEXT: Flags [ +# CHECK-FLAGS-NEXT: SHF_COMPRESSED +# CHECK-FLAGS-NEXT: ] +# CHECK-FLAGS-NOT: Name: .debug_foo + +# NOTE: Zlib compression makes things worse in this case. +# SIZE-ORIGINAL: 8 +# SIZE-COMPRESSED: 40 + Index: tools/llvm-objcopy/ObjcopyOpts.td =================================================================== --- tools/llvm-objcopy/ObjcopyOpts.td +++ tools/llvm-objcopy/ObjcopyOpts.td @@ -17,6 +17,9 @@ defm output_target : Eq<"output-target">, HelpText<"Format of the output file">, Values<"binary">; +defm compress_debug_sections : Eq<"compress-debug-sections">, + MetaVarName<"[ none | zlib | zlib-gnu ]">, + HelpText<"Compress DWARF debug sections using specified style. Supported styles: 'zlib-gnu' and 'zlib'">; def O : JoinedOrSeparate<["-"], "O">, Alias; defm split_dwo : Eq<"split-dwo">, Index: tools/llvm-objcopy/Object.h =================================================================== --- tools/llvm-objcopy/Object.h +++ tools/llvm-objcopy/Object.h @@ -14,8 +14,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/JamCRC.h" #include @@ -39,6 +41,7 @@ class GnuDebugLinkSection; class GroupSection; class SectionIndexSection; +class CompressedSection; class Segment; class Object; struct Symbol; @@ -86,6 +89,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) = 0; virtual void visit(const GroupSection &Sec) = 0; virtual void visit(const SectionIndexSection &Sec) = 0; + virtual void visit(const CompressedSection &Sec) = 0; }; class SectionWriter : public SectionVisitor { @@ -104,6 +108,7 @@ virtual void visit(const GnuDebugLinkSection &Sec) override = 0; virtual void visit(const GroupSection &Sec) override = 0; virtual void visit(const SectionIndexSection &Sec) override = 0; + virtual void visit(const CompressedSection &Sec) override = 0; explicit SectionWriter(Buffer &Buf) : Out(Buf) {} }; @@ -122,6 +127,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit ELFSectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -139,6 +145,7 @@ void visit(const GnuDebugLinkSection &Sec) override; void visit(const GroupSection &Sec) override; void visit(const SectionIndexSection &Sec) override; + void visit(const CompressedSection &Sec) override; explicit BinarySectionWriter(Buffer &Buf) : SectionWriter(Buf) {} }; @@ -246,7 +253,7 @@ class SectionBase { public: - StringRef Name; + std::string Name; Segment *ParentSegment = nullptr; uint64_t HeaderOffset; uint64_t OriginalOffset = std::numeric_limits::max(); @@ -265,6 +272,25 @@ uint64_t Type = ELF::SHT_NULL; ArrayRef OriginalData; + SectionBase &operator=(const SectionBase &rhs) { + Name = rhs.Name; + ParentSegment = rhs.ParentSegment; + HeaderOffset = rhs.HeaderOffset; + OriginalOffset = std::numeric_limits::max(); + Index = rhs.Index; + HasSymbol = rhs.HasSymbol; + Addr = rhs.Addr; + Align = rhs.Align; + EntrySize = rhs.EntrySize; + Flags = rhs.Flags; + Info = rhs.Info; + Link = rhs.Link; + NameIndex = rhs.NameIndex; + Offset = rhs.Offset; + Type = ELF::SHT_PROGBITS; + return *this; + } + virtual ~SectionBase() = default; virtual void initialize(SectionTableRef SecTable); @@ -341,7 +367,7 @@ public: OwnedDataSection(StringRef SecName, ArrayRef Data) : Data(std::begin(Data), std::end(Data)) { - Name = SecName; + Name = SecName.str(); Type = ELF::SHT_PROGBITS; Size = Data.size(); OriginalOffset = std::numeric_limits::max(); @@ -350,6 +376,19 @@ void accept(SectionVisitor &Sec) const override; }; +class CompressedSection : public SectionBase { + MAKE_SEC_WRITER_FRIEND + + std::vector Data; + SmallVector CompressedData; + DebugCompressionType CompressionType; + +public: + CompressedSection(const SectionBase &Sec, + DebugCompressionType CompressionType); + void accept(SectionVisitor &Visitor) const override; +}; + // There are two types of string tables that can exist, dynamic and not dynamic. // In the dynamic case the string table is allocated. Changing a dynamic string // table would mean altering virtual addresses and thus the memory image. So Index: tools/llvm-objcopy/Object.cpp =================================================================== --- tools/llvm-objcopy/Object.cpp +++ tools/llvm-objcopy/Object.cpp @@ -138,6 +138,68 @@ Visitor.visit(*this); } +void BinarySectionWriter::visit(const CompressedSection &Sec) { + error("Cannot write compressed section '" + Sec.Name + "' "); +} + +template +void ELFSectionWriter::visit(const CompressedSection &Sec) { + uint8_t *Buf = Out.getBufferStart(); + Buf += Sec.Offset; + + const uint64_t DecompressedSize = Sec.OriginalData.size(); + if (Sec.CompressionType == DebugCompressionType::GNU) { + ArrayRef Magic = {'Z', 'L', 'I', 'B'}; + std::copy(Magic.begin(), Magic.end(), Buf); + Buf += Magic.size(); + uint64_t *DecompressedSizePtr = reinterpret_cast(Buf); + *DecompressedSizePtr = support::endian::read64be(&DecompressedSize); + Buf += sizeof(DecompressedSize); + } else { + auto Chdr = reinterpret_cast *>(Buf); + Chdr->ch_type = ELF::ELFCOMPRESS_ZLIB; + Chdr->ch_size = DecompressedSize; + Chdr->ch_addralign = Sec.Align; + Buf += sizeof(*Chdr); + } + + std::copy(Sec.CompressedData.begin(), Sec.CompressedData.end(), Buf); +} + +CompressedSection::CompressedSection(const SectionBase &Sec, + DebugCompressionType CompressionType) + : Data(std::begin(Sec.OriginalData), std::end(Sec.OriginalData)), + CompressionType(CompressionType) { + + *(static_cast(this)) = Sec; + + OriginalData = ArrayRef(Data.data(), Data.size()); + + if (Error E = zlib::compress( + StringRef(reinterpret_cast(OriginalData.data()), + OriginalData.size()), + CompressedData)) + reportError(Name, std::move(E)); + + size_t ChdrSize; + if (CompressionType == DebugCompressionType::GNU) { + Name = std::string(".z") + Sec.Name.substr(1); + ChdrSize = sizeof("ZLIB") - 1 + sizeof(uint64_t); + } else { + Flags |= ELF::SHF_COMPRESSED; + ChdrSize = + std::max(std::max(sizeof(object::Elf_Chdr_Impl), + sizeof(object::Elf_Chdr_Impl)), + std::max(sizeof(object::Elf_Chdr_Impl), + sizeof(object::Elf_Chdr_Impl))); + } + Size = ChdrSize + CompressedData.size(); +} + +void CompressedSection::accept(SectionVisitor &Visitor) const { + Visitor.visit(*this); +} + void StringTableSection::addString(StringRef Name) { StrTabBuilder.add(Name); Size = StrTabBuilder.getSize(); Index: tools/llvm-objcopy/llvm-objcopy.cpp =================================================================== --- tools/llvm-objcopy/llvm-objcopy.cpp +++ tools/llvm-objcopy/llvm-objcopy.cpp @@ -175,6 +175,7 @@ bool StripSections = false; bool StripUnneeded = false; bool Weaken = false; + DebugCompressionType CompressionType = DebugCompressionType::None; }; using SectionPred = std::function; @@ -290,12 +291,12 @@ } static bool isDebugSection(const SectionBase &Sec) { - return Sec.Name.startswith(".debug") || Sec.Name.startswith(".zdebug") || - Sec.Name == ".gdb_index"; + return StringRef(Sec.Name).startswith(".debug") || + StringRef(Sec.Name).startswith(".zdebug") || Sec.Name == ".gdb_index"; } static bool isDWOSection(const SectionBase &Sec) { - return Sec.Name.endswith(".dwo"); + return StringRef(Sec.Name).endswith(".dwo"); } static bool onlyKeepDWOPred(const Object &Obj, const SectionBase &Sec) { @@ -405,6 +406,30 @@ object_error::parse_failed); } +static bool isCompressed(const SectionBase &Section) { + return StringRef(Section.Name).startswith(".zdebug") || + (Section.Flags & ELF::SHF_COMPRESSED); +} + +static void appendCompressableSections(const CopyConfig &Config, Object &Obj, + SectionPred &RemovePred) { + std::vector SectionsToRemove; + std::for_each(Obj.sections().begin(), Obj.sections().end(), + [&SectionsToRemove](SectionBase &Section) { + if (!isCompressed(Section) && isDebugSection(Section) && + Section.Name != ".gdb_index") { + SectionsToRemove.push_back(&Section); + } + }); + + for (auto S : SectionsToRemove) { + const SectionBase &Section = *S; + Obj.addSection(Section, Config.CompressionType); + RemovePred = [RemovePred, &Section](const SectionBase &Sec) { + return &Sec == &Section || RemovePred(Sec); + }; + } +} // This function handles the high level operations of GNU objcopy including // handling command line options. It's important to outline certain properties // we expect to hold of the command line operations. Any operation that "keeps" @@ -564,7 +589,7 @@ return true; if (&Sec == Obj.SectionNames) return false; - if (Sec.Name.startswith(".gnu.warning")) + if (StringRef(Sec.Name).startswith(".gnu.warning")) return false; return (Sec.Flags & SHF_ALLOC) == 0; }; @@ -616,6 +641,9 @@ }; } + if (Config.CompressionType != DebugCompressionType::None) + appendCompressableSections(Config, Obj, RemovePred); + Obj.removeSections(RemovePred); if (!Config.SectionsToRename.empty()) { @@ -859,6 +887,19 @@ Config.BinaryArch = getMachineInfo(BinaryArch); } + Config.CompressionType = + StringSwitch( + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)) + .Case("zlib-gnu", DebugCompressionType::GNU) + .Case("zlib", DebugCompressionType::Z) + .Default(DebugCompressionType::None); + + if (Config.CompressionType == DebugCompressionType::None && + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections) != "") { + error("Invalid or unsupported --compress-debug-sections format: " + + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)); + } + Config.SplitDWO = InputArgs.getLastArgValue(OBJCOPY_split_dwo); Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink); Config.SymbolsPrefix = InputArgs.getLastArgValue(OBJCOPY_prefix_symbols);