Index: include/llvm/Object/Compressor.h =================================================================== --- /dev/null +++ include/llvm/Object/Compressor.h @@ -0,0 +1,92 @@ +//===-- Compressor.h --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===-----------------------------------------------------------------------===/ + +#ifndef LLVM_OBJECT_COMPRESSOR_H +#define LLVM_OBJECT_COMPRESSOR_H + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/EndianStream.h" + +namespace llvm { +namespace object { + +/// Return the gnu style compressed section name. +StringRef getDebugSectionName(const StringRef Name, bool IsGnuStyle); + +/// Returns if the section can be compressed based on its name (must have a +/// debug name, starts with .*debug. +bool isCompressable(StringRef Name); + +/// Returns if the section is already compressed based on the section contents +/// Name and Flags. +bool isCompressed(StringRef Name, StringRef Contents, uint64_t Flags); + +template +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, unsigned Align); + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align); + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align); + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align); + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align); + +/// Compressor helps to handle compression of compressed sections. +template class Compressor { +public: + void writeHeader(support::endian::Writer &W, uint64_t DecompressedSize, + unsigned Alignment, bool IsGnuStyle) { + if (IsGnuStyle) { + const StringRef Magic = "ZLIB"; + W.OS << Magic; + support::endian::write(W.OS, DecompressedSize, support::big); + return; + } + + produceCompressedZLibHeader(W, DecompressedSize, Alignment); + } + + /// Uncompress section data to raw buffer provided. + /// @param W Destination buffer stream. + Error writeCompressedSectionData(support::endian::Writer &W) { + SmallVector CompressedBuffer; + auto E = zlib::compress(SectionData, CompressedBuffer); + W.OS << StringRef(CompressedBuffer.data(), CompressedBuffer.size()).str(); + return E; + } + + explicit Compressor(StringRef Data) : SectionData(Data) {} + +private: + StringRef SectionData; +}; + +} // end namespace object +} // end namespace llvm + +#endif // LLVM_OBJECT_COMPRESSOR_H Index: lib/Object/CMakeLists.txt =================================================================== --- lib/Object/CMakeLists.txt +++ lib/Object/CMakeLists.txt @@ -21,6 +21,7 @@ SymbolSize.cpp WasmObjectFile.cpp WindowsResource.cpp + Compressor.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Object Index: lib/Object/Compressor.cpp =================================================================== --- /dev/null +++ lib/Object/Compressor.cpp @@ -0,0 +1,113 @@ +//===-- Compressor.cpp ----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Object/Compressor.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" + +using namespace llvm; +using namespace llvm::support::endian; +using namespace object; + +namespace llvm { +namespace object { + +StringRef getDebugSectionName(const StringRef Name, bool IsGnuStyle) { + + if (!Name.startswith(".debug") && !Name.startswith(".zdebug")) + return ""; + + std::string LookupName = Name.substr(Name.startswith(".debug") ? 1 : 2).str(); + std::string Dot("."); + + static std::map> NameMap; + auto I = NameMap.find(LookupName); + if (I == NameMap.end()) { + NameMap.insert(std::pair>( + LookupName, + std::make_tuple(Dot + LookupName, (Dot + "z") + LookupName))); + I = NameMap.find(LookupName); + } + + StringRef debugName, zdebugName; + std::tie(debugName, zdebugName) = I->second; + return IsGnuStyle ? zdebugName : debugName; +} + +bool isCompressable(StringRef Name) { + StringRef DotDebugName = getDebugSectionName(Name, false /* IsGnuStyle */); + return DotDebugName.startswith(".debug"); +} + +bool isCompressed(StringRef Name, StringRef Contents, uint64_t Flags) { + if (!Name.startswith(".debug") && !Name.startswith(".zdebug")) + return false; + + StringRef magic = Contents.substr(4); + return Name.startswith(".zdebug") || (magic == "ZLIB") || + (Flags & ELF::SHF_COMPRESSED); +} + +template +void writeZLibHeaderReserved(support::endian::Writer &W) {} + +template <> +void writeZLibHeaderReserved(support::endian::Writer &W) { + W.write(static_cast(0)); // ch_reserved field. +} + +template +void produceCompressedZLibHeader2(support::endian::Writer &W, + uint64_t DecompressedSize, unsigned Align) { + T Elf_Chr; + W.write(static_cast(ELF::ELFCOMPRESS_ZLIB)); + writeZLibHeaderReserved(W); + W.write(static_cast(DecompressedSize)); + W.write(static_cast(Align)); +} + +template +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, unsigned Align) {} + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align) { + produceCompressedZLibHeader2(W, DecompressedSize, Align); +} + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align) { + produceCompressedZLibHeader2(W, DecompressedSize, Align); +} + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align) { + produceCompressedZLibHeader2(W, DecompressedSize, Align); +} + +template <> +void produceCompressedZLibHeader(support::endian::Writer &W, + uint64_t DecompressedSize, + unsigned Align) { + produceCompressedZLibHeader2(W, DecompressedSize, Align); +} + +} // end namespace object +} // end namespace llvm Index: tools/llvm-objcopy/ObjcopyOpts.td =================================================================== --- tools/llvm-objcopy/ObjcopyOpts.td +++ tools/llvm-objcopy/ObjcopyOpts.td @@ -16,6 +16,11 @@ defm output_target : Eq<"output-target">, HelpText<"Format of the output file">, Values<"binary">; +defm compress_debug_sections : Eq<"compress-debug-sections">, + MetaVarName<"[ none | zlib | zlib-gnu (deprecated) ]">, + HelpText<"Enable zlib-gnu or zlib Compression of DWARF debug sections.">; +def decompress_debug_sections : Flag<["-", "--"], "decompress-debug-sections">, + HelpText<"Decompress DWARF debug sections.">; def O : JoinedOrSeparate<["-"], "O">, Alias; defm split_dwo : Eq<"split-dwo">, Index: tools/llvm-objcopy/Object.h =================================================================== --- tools/llvm-objcopy/Object.h +++ tools/llvm-objcopy/Object.h @@ -14,7 +14,10 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCTargetOptions.h" #include "llvm/MC/StringTableBuilder.h" +#include "llvm/Object/Compressor.h" +#include "llvm/Object/Decompressor.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/JamCRC.h" @@ -43,6 +46,44 @@ class Object; struct Symbol; +struct CopyConfig { + StringRef OutputFilename; + StringRef InputFilename; + StringRef OutputFormat; + StringRef InputFormat; + StringRef BinaryArch; + + StringRef SplitDWO; + StringRef AddGnuDebugLink; + std::vector ToRemove; + std::vector Keep; + std::vector OnlyKeep; + std::vector AddSection; + std::vector SymbolsToLocalize; + std::vector SymbolsToGlobalize; + std::vector SymbolsToWeaken; + std::vector SymbolsToRemove; + std::vector SymbolsToKeep; + StringMap SectionsToRename; + StringMap SymbolsToRename; + bool StripAll = false; + bool StripAllGNU = false; + bool StripDebug = false; + bool StripSections = false; + bool StripNonAlloc = false; + bool StripDWO = false; + bool StripUnneeded = false; + bool ExtractDWO = false; + bool LocalizeHidden = false; + bool Weaken = false; + bool DiscardAll = false; + bool OnlyKeepDebug = false; + bool KeepFileSymbols = false; + bool DecompressDebugSections = false; + + DebugCompressionType CompressDebugSections = DebugCompressionType::None; +}; + class SectionTableRef { MutableArrayRef> Sections; @@ -190,6 +231,9 @@ template class ELFWriter : public Writer { private: + DebugCompressionType CompressDebugSections = DebugCompressionType::None; + bool DecompressDebugSections = false; + using Elf_Shdr = typename ELFT::Shdr; using Elf_Phdr = typename ELFT::Phdr; using Elf_Ehdr = typename ELFT::Ehdr; @@ -214,8 +258,11 @@ void finalize() override; void write() override; - ELFWriter(Object &Obj, Buffer &Buf, bool WSH) - : Writer(Obj, Buf), WriteSectionHeaders(WSH) {} + ELFWriter(Object &Obj, Buffer &Buf, const CopyConfig &Config) + : Writer(Obj, Buf), WriteSectionHeaders(!Config.StripSections) { + this->CompressDebugSections = Config.CompressDebugSections; + this->DecompressDebugSections = Config.DecompressDebugSections; + } }; class BinaryWriter : public Writer { @@ -259,6 +306,13 @@ virtual void removeSymbols(function_ref ToRemove); virtual void accept(SectionVisitor &Visitor) const = 0; virtual void markSymbols(); + virtual ArrayRef *getContents() { + assert(false && "Section has no Contents."); + return nullptr; + } + virtual void updateContents(std::unique_ptr> Contents) { + assert(false && "Section has no Contents."); + } }; class Segment { @@ -308,6 +362,7 @@ MAKE_SEC_WRITER_FRIEND ArrayRef Contents; + std::unique_ptr> ModifiedContents; SectionBase *LinkSection = nullptr; public: @@ -317,6 +372,8 @@ void removeSectionReferences(const SectionBase *Sec) override; void initialize(SectionTableRef SecTable) override; void finalize() override; + ArrayRef *getContents() override { return &Contents; } + void updateContents(std::unique_ptr> Update) override; }; class OwnedDataSection : public SectionBase { Index: tools/llvm-objcopy/Object.cpp =================================================================== --- tools/llvm-objcopy/Object.cpp +++ tools/llvm-objcopy/Object.cpp @@ -491,6 +491,13 @@ void Section::finalize() { this->Link = LinkSection ? LinkSection->Index : 0; } +void Section::updateContents(std::unique_ptr> Update) { + ModifiedContents = std::move(Update); + Size = ModifiedContents->size(); + const uint8_t *ContentsPtr = (const uint8_t *)ModifiedContents->data(); + Contents = ArrayRef(ContentsPtr, Size); +} + void GnuDebugLinkSection::init(StringRef File, StringRef Data) { FileName = sys::path::filename(File); // The format for the .gnu_debuglink starts with the file name and is @@ -1213,6 +1220,51 @@ reportError(Buf.getName(), errorToErrorCode(std::move(E))); } +template +std::unique_ptr> decompress(StringRef &Name, + StringRef Contents) { + + auto ModifiedContents = make_unique>(); + + bool Is64Bit = T::Is64Bits; + bool IsLittle = (T::TargetEndianness == endianness::little); + auto D = object::Decompressor::create(Name, Contents, IsLittle, Is64Bit); + if (!D) + consumeError(D.takeError()); + + auto E = D->resizeAndDecompress(*ModifiedContents.get()); + consumeError(std::move(E)); + + if (const bool IsGnuStyle = Name.startswith(".zdebug")) + Name = object::getDebugSectionName(Name, !IsGnuStyle); + + return std::move(ModifiedContents); +} + +template +std::unique_ptr> +compress(StringRef &Name, StringRef Contents, uint64_t Align, bool IsGnuStyle) { + + auto ModifiedContents = make_unique>(); + + object::endianness E = T::TargetEndianness; + raw_svector_ostream OS(*ModifiedContents.get()); + support::endian::Writer W(OS, E); + + auto C = object::Compressor(Contents); + C.writeHeader(W, Contents.size(), Align, IsGnuStyle); + auto Error = C.writeCompressedSectionData(W); + consumeError(std::move(Error)); + + if (ModifiedContents->size() < Contents.size()) { + if (IsGnuStyle) + Name = object::getDebugSectionName(Name, IsGnuStyle); + return std::move(ModifiedContents); + } + + return nullptr; +} + template void ELFWriter::finalize() { // It could happen that SectionNames has been removed and yet the user wants // a section header table output. We need to throw an error if a user tries @@ -1256,10 +1308,37 @@ } } + bool doDecompress = DecompressDebugSections; + bool IsGnuStyle = (CompressDebugSections == DebugCompressionType::GNU); + bool doCompress = (CompressDebugSections != DebugCompressionType::None) && + !DecompressDebugSections; + // Make sure we add the names of all the sections. Importantly this must be // done after we decide to add or remove SectionIndexes. if (Obj.SectionNames != nullptr) - for (const auto &Section : Obj.sections()) { + for (auto &Section : Obj.sections()) { + + bool isCompressable = object::isCompressable(Section.Name); + if (isCompressable && (doCompress || doDecompress)) { + StringRef Contents((const char *)Section.getContents()->data(), + Section.getContents()->size()); + + bool isCompressed = + object::isCompressed(Section.Name, Contents, Section.Flags); + if (doDecompress && isCompressed) { + auto ModifiedContents = decompress(Section.Name, Contents); + Section.updateContents(std::move(ModifiedContents)); + } else if (doCompress && !isCompressed) { + auto ModifiedContents = + compress(Section.Name, Contents, Section.Align, IsGnuStyle); + if (ModifiedContents != nullptr) { + Section.updateContents(std::move(ModifiedContents)); + if (!IsGnuStyle) + Section.Flags |= ELF::SHF_COMPRESSED; + } + } + } + Obj.SectionNames->addString(Section.Name); } Index: tools/llvm-objcopy/llvm-objcopy.cpp =================================================================== --- tools/llvm-objcopy/llvm-objcopy.cpp +++ tools/llvm-objcopy/llvm-objcopy.cpp @@ -115,41 +115,6 @@ StripOptTable() : OptTable(StripInfoTable, true) {} }; -struct CopyConfig { - StringRef OutputFilename; - StringRef InputFilename; - StringRef OutputFormat; - StringRef InputFormat; - StringRef BinaryArch; - - StringRef SplitDWO; - StringRef AddGnuDebugLink; - std::vector ToRemove; - std::vector Keep; - std::vector OnlyKeep; - std::vector AddSection; - std::vector SymbolsToLocalize; - std::vector SymbolsToGlobalize; - std::vector SymbolsToWeaken; - std::vector SymbolsToRemove; - std::vector SymbolsToKeep; - StringMap SectionsToRename; - StringMap SymbolsToRename; - bool StripAll = false; - bool StripAllGNU = false; - bool StripDebug = false; - bool StripSections = false; - bool StripNonAlloc = false; - bool StripDWO = false; - bool StripUnneeded = false; - bool ExtractDWO = false; - bool LocalizeHidden = false; - bool Weaken = false; - bool DiscardAll = false; - bool OnlyKeepDebug = false; - bool KeepFileSymbols = false; -}; - using SectionPred = std::function; } // namespace @@ -211,17 +176,13 @@ // Depending on the initial ELFT and OutputFormat we need a different Writer. switch (OutputElfType) { case ELFT_ELF32LE: - return llvm::make_unique>(Obj, Buf, - !Config.StripSections); + return llvm::make_unique>(Obj, Buf, Config); case ELFT_ELF64LE: - return llvm::make_unique>(Obj, Buf, - !Config.StripSections); + return llvm::make_unique>(Obj, Buf, Config); case ELFT_ELF32BE: - return llvm::make_unique>(Obj, Buf, - !Config.StripSections); + return llvm::make_unique>(Obj, Buf, Config); case ELFT_ELF64BE: - return llvm::make_unique>(Obj, Buf, - !Config.StripSections); + return llvm::make_unique>(Obj, Buf, Config); } llvm_unreachable("Invalid output format"); } @@ -587,6 +548,13 @@ Config.OutputFormat = InputArgs.getLastArgValue(OBJCOPY_output_target); Config.BinaryArch = InputArgs.getLastArgValue(OBJCOPY_binary_architecture); + Config.CompressDebugSections = + StringSwitch( + InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections)) + .Case("zlib-gnu", DebugCompressionType::GNU) + .Case("zlib", DebugCompressionType::Z) + .Default(DebugCompressionType::None); + Config.SplitDWO = InputArgs.getLastArgValue(OBJCOPY_split_dwo); Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink); @@ -627,6 +595,7 @@ Config.DiscardAll = InputArgs.hasArg(OBJCOPY_discard_all); Config.OnlyKeepDebug = InputArgs.hasArg(OBJCOPY_only_keep_debug); Config.KeepFileSymbols = InputArgs.hasArg(OBJCOPY_keep_file_symbols); + Config.DecompressDebugSections = InputArgs.hasArg(OBJCOPY_decompress_debug_sections); for (auto Arg : InputArgs.filtered(OBJCOPY_localize_symbol)) Config.SymbolsToLocalize.push_back(Arg->getValue()); for (auto Arg : InputArgs.filtered(OBJCOPY_globalize_symbol)) @@ -638,6 +607,12 @@ for (auto Arg : InputArgs.filtered(OBJCOPY_keep_symbol)) Config.SymbolsToKeep.push_back(Arg->getValue()); + if (Config.DecompressDebugSections && + Config.CompressDebugSections != DebugCompressionType::None) { + error("Can not specify --compress-debug-sections as well as " + "--decompress-debug-sections at the same time."); + } + return Config; }