[ELF] Support ELFCOMPRESS_ZSTD input

[MaskRay]

Summary

so that lld accepts relocatable object files produced by clang -c -g -gz=zstd.

We don't want to increase the size of InputSection, so do redundant but cheap
ch_type checks instead.

[MaskRay]

rebase

[MaskRay]

Fix gdb-index

[MaskRay]

add release note

[MaskRay]

simplify test

[dblaikie]

Looks like this didn't get reviewed?

[MaskRay]

In D129406#3780891, @dblaikie wrote:

Looks like this didn't get reviewed?

Happy to address changes if @ikudrin or @peter.smith has any suggestion.
The changes are pretty mechanical and a dependent was accepted.

[dblaikie]

In D129406#3780896, @MaskRay wrote:

In D129406#3780891, @dblaikie wrote:

Looks like this didn't get reviewed?

Happy to address changes if @ikudrin or @peter.smith has any suggestion.
The changes are pretty mechanical and a dependent was accepted.

The LLVM review process is that if something's sent for review it should not be committed until that review is completed ( https://llvm.org/docs/CodeReview.html#code-review-workflow "Specifically, once a patch is sent out for review, it needs an explicit approval before it is committed. Do not assume silent approval, or solicit objections to a patch with a deadline." ) . (we don't want to encourage people to send things for review that they believe needs review, then commit due to lack of feedback in the time they'd like to receive it)

lld/ELF/InputSection.cpp

 size_t InputSectionBase::getSize() const {
   if (auto *s = dyn_cast<SyntheticSection>(this))
     return s->getSize();
   if (uncompressedSize >= 0)
     return uncompressedSize;
   return rawData.size() - bytesDropped;
 }
 
+template <class ELFT>
+static void decompressAux(const InputSectionBase &sec, uint8_t *out,
+                          size_t size) {
+  auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(sec.rawData.data());
+  auto compressed = sec.rawData.slice(sizeof(typename ELFT::Chdr));
+  if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
+                    ? compression::zlib::uncompress(compressed, out, size)
+                    : compression::zstd::uncompress(compressed, out, size))
+    fatal(toString(&sec) +
+          ": decompress failed: " + llvm::toString(std::move(e)));
+}
+
 void InputSectionBase::decompress() const {
   size_t size = uncompressedSize;
   uint8_t *uncompressedBuf;
   {
     static std::mutex mu;
     std::lock_guard<std::mutex> lock(mu);
     uncompressedBuf = bAlloc().Allocate<uint8_t>(size);
   }
 
-  if (Error e = compression::zlib::uncompress(rawData, uncompressedBuf, size))
-    fatal(toString(this) +
-          ": decompress failed: " + llvm::toString(std::move(e)));
+  invokeELFT(decompressAux, *this, uncompressedBuf, size);
   rawData = makeArrayRef(uncompressedBuf, size);
   uncompressedSize = -1;
 }
 
 template <class ELFT> RelsOrRelas<ELFT> InputSectionBase::relsOrRelas() const {
   if (relSecIdx == 0)
     return {};
   RelsOrRelas<ELFT> ret;
if (rawData.size() < sizeof(typename ELFT::Chdr)) {
     return;
   }
 
   auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(rawData.data());
   if (hdr->ch_type == ELFCOMPRESS_ZLIB) {
     if (!compression::zlib::isAvailable())
       error(toString(this) + " is compressed with ELFCOMPRESS_ZLIB, but lld is "
                              "not built with zlib support");
+  } else if (hdr->ch_type == ELFCOMPRESS_ZSTD) {
+    if (!compression::zstd::isAvailable())
+      error(toString(this) + " is compressed with ELFCOMPRESS_ZSTD, but lld is "
+                             "not built with zstd support");
   } else {
     error(toString(this) + ": unsupported compression type (" +
           Twine(hdr->ch_type) + ")");
     return;
   }
 
   uncompressedSize = hdr->ch_size;
   alignment = std::max<uint32_t>(hdr->ch_addralign, 1);
-  rawData = rawData.slice(sizeof(*hdr));
 }
 
 InputSection *InputSectionBase::getLinkOrderDep() const {
   assert(flags & SHF_LINK_ORDER);
   if (!link)
     return nullptr;
   return cast<InputSection>(file->getSections()[link]);
 }
template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
   if (LLVM_UNLIKELY(type == SHT_GROUP)) {
     copyShtGroup<ELFT>(buf);
     return;
   }
 
   // If this is a compressed section, uncompress section contents directly
   // to the buffer.
   if (uncompressedSize >= 0) {
+    auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(rawData.data());
+    auto compressed = rawData.slice(sizeof(typename ELFT::Chdr));
     size_t size = uncompressedSize;
-    if (Error e = compression::zlib::uncompress(rawData, buf, size))
+    if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
+                      ? compression::zlib::uncompress(compressed, buf, size)
+                      : compression::zstd::uncompress(compressed, buf, size))
       fatal(toString(this) +
             ": decompress failed: " + llvm::toString(std::move(e)));
     uint8_t *bufEnd = buf + size;
     relocate<ELFT>(buf, bufEnd);
     return;
   }
 
   // Copy section contents from source object file to output file

lld/docs/ReleaseNotes.rst

 from the `LLVM releases web site <https://llvm.org/releases/>`_.
 
 Non-comprehensive list of changes in this release
 =================================================
 
 ELF Improvements
 ----------------
 
+* ``ELFCOMPRESS_ZSTD`` compressed input sections are now supported.
+  (`D129406 <https://reviews.llvm.org/D129406>`_)
+
 Breaking changes
 ----------------
 
 COFF Improvements
 -----------------
 
 * ...
 

lld/test/CMakeLists.txt

 llvm_canonicalize_cmake_booleans(
   ENABLE_BACKTRACES
   LLVM_ENABLE_ZLIB
+  LLVM_ENABLE_ZSTD
   LLVM_ENABLE_LIBXML2
   LLD_DEFAULT_LD_LLD_IS_MINGW
   LLVM_HAVE_LIBXAR
   LLVM_BUILD_EXAMPLES
   LLVM_ENABLE_PLUGINS
   LLVM_BYE_LINK_INTO_TOOLS
   )
 

lld/test/ELF/compress-debug-sections-zstd.s

+# REQUIRES: x86, zstd
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 --compress-debug-sections=zstd %s -o %t.o
+
+# RUN: ld.lld %t.o -o %t.so -shared
+# RUN: llvm-readelf -S -x .debug_str %t.so | FileCheck %s
+
+# CHECK:      .debug_str    PROGBITS [[#%x,]] [[#%x,]] [[#%x,]] 01 MS  0 0  1
+# CHECK:      Hex dump of section '.debug_str':
+# CHECK-NEXT: 0x00000000 756e7369 676e6564 20696e74 00636861 unsigned int.cha
+# CHECK-NEXT: 0x00000010 7200756e 7369676e 65642063 68617200 r.unsigned char.
+# CHECK-NEXT: 0x00000020 73686f72 7420756e 7369676e 65642069 short unsigned i
+# CHECK-NEXT: 0x00000030 6e74006c 6f6e6720 756e7369 676e6564 nt.long unsigned
+# CHECK-NEXT: 0x00000040 20696e74 00                          int.
+
+.section .debug_str,"MS",@progbits,1
+.LASF2:
+ .string "short unsigned int"
+.LASF3:
+ .string "unsigned int"
+.LASF0:
+ .string "long unsigned int"
+.LASF8:
+ .string "char"
+.LASF1:
+ .string "unsigned char"

lld/test/ELF/compressed-input-err-zstd.s

+# UNSUPPORTED: zstd
+# RUN: yaml2obj %s -o %t.o
+# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+
+# CHECK: error: {{.*}}.o:(.debug_info) is compressed with ELFCOMPRESS_ZSTD, but lld is not built with zstd support
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Type:         SHT_PROGBITS
+    Name:         .debug_info
+    Flags:        [ SHF_COMPRESSED ]
+    AddressAlign: 8
+    Content:      "020000000000000000000000000000000100000000000000789c030000000001"

lld/test/ELF/gdb-index.s

 # RUN: llvm-mc -compress-debug-sections=zlib -filetype=obj -triple=x86_64-pc-linux \
 # RUN:   %p/Inputs/gdb-index.s -o %t2.o
 # RUN: ld.lld --gdb-index %t1.o %t2.o -o %t
 
 # RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=DISASM
 # RUN: llvm-dwarfdump -gdb-index %t | FileCheck %s --check-prefix=DWARF
 # RUN: llvm-readelf -S %t | FileCheck %s --check-prefix=SECTION
 
+# RUN: %if zstd %{ llvm-mc -compress-debug-sections=zlib -filetype=obj -triple=x86_64 %p/Inputs/gdb-index.s -o %t2.o %}
+# RUN: %if zstd %{ ld.lld --gdb-index %t1.o %t2.o -o %t %}
+# RUN: %if zstd %{ llvm-objdump -d %t | FileCheck %s --check-prefix=DISASM %}
+# RUN: %if zstd %{ llvm-dwarfdump --gdb-index %t | FileCheck %s --check-prefix=DWARF %}
+# RUN: %if zstd %{ llvm-readelf -S %t | FileCheck %s --check-prefix=SECTION %}
+
 # DISASM:       Disassembly of section .text:
 # DISASM-EMPTY:
 # DISASM:       <entrypoint>:
 # DISASM-CHECK:   201000: 90 nop
 # DISASM-CHECK:   201001: cc int3
 # DISASM-CHECK:   201002: cc int3
 # DISASM-CHECK:   201003: cc int3
 # DISASM:       <aaaaaaaaaaaaaaaa>:

lld/test/lit.site.cfg.py.in

 config.enable_backtrace = @ENABLE_BACKTRACES@
 config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@"
 config.lld_obj_root = "@LLD_BINARY_DIR@"
 config.lld_libs_dir = lit_config.substitute("@CURRENT_LIBS_DIR@")
 config.lld_tools_dir = lit_config.substitute("@CURRENT_TOOLS_DIR@")
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.sizeof_void_p = @CMAKE_SIZEOF_VOID_P@
 config.ld_lld_default_mingw = @LLD_DEFAULT_LD_LLD_IS_MINGW@
 config.build_examples = @LLVM_BUILD_EXAMPLES@
 config.has_plugins = @LLVM_ENABLE_PLUGINS@
 config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@LLD_SOURCE_DIR@/test/lit.cfg.py")