[lldb] Add zstd support

[ckissane]

Summary

adds zstd support to Decompressor so lldb can work with zstd

[MaskRay]

lldb/test/Shell/ObjectFile/ELF/compressed-sections-zstd.yaml

19

Delete .zdebug. It is unrelated to zstd

26

Align

llvm/lib/Object/Decompressor.cpp

48

Use a switch so that in the case of new compression format, the code get notified due to -Wswitch

52

Just use the return type of getReasonIfUnsupported as the error message

74

[labath]

Seems fine to me, though you may want to have a llvm test for the new functionality, given that the patch is exclusively changing llvm code.

[MaskRay]

Ping for unresolved issues.

[treapster]

It seems like the new code doesn't check zstd version, thus breaking the build if you have something below 1.4.0.

lldb/test/CMakeLists.txt

 
 # These values are not canonicalized within LLVM.
 llvm_canonicalize_cmake_booleans(
   LLDB_BUILD_INTEL_PT
   LLDB_ENABLE_PYTHON
   LLDB_ENABLE_LUA
   LLDB_ENABLE_LZMA
   LLVM_ENABLE_ZLIB
+  LLVM_ENABLE_ZSTD
   LLVM_ENABLE_SHARED_LIBS
   LLDB_HAS_LIBCXX
   LLDB_USE_SYSTEM_DEBUGSERVER
   LLDB_IS_64_BITS)
 
 # Configure the individual test suites.
 add_subdirectory(API)
 add_subdirectory(Shell)

lldb/test/Shell/ObjectFile/ELF/compressed-sections-zstd.yaml

+# REQUIRES: zstd
+# RUN: yaml2obj %s -o %t
+# RUN: lldb-test object-file --contents %t | FileCheck %s
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_386
+Sections:
+  - Name:            .hello_elf
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_COMPRESSED ]
+    Content:         02000000080000000100000028B52FFD20084100002030405060708090
+  - Name:            .bogus
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_COMPRESSED ]
+    Content:         deadbeefbaadf00d
+## The legacy .zdebug format is not supported.
+  - Name:            .zdebug_info
+    Type:            SHT_PROGBITS
+    Content:         5A4C49420000000000000008789c5330700848286898000009c802c1
+
+# CHECK: Name: .hello_elf
+# CHECK-NEXT: Type: regular
+# CHECK: VM address: 0
+# CHECK-NEXT: VM size: 0
+# CHECK-NEXT: File size: 29
+# CHECK-NEXT: Data: (
+# CHECK-NEXT: 20304050 60708090
+# CHECK-NEXT: )
+
+# CHECK: Name: .bogus
+# CHECK-NEXT: Type: regular
+# CHECK: VM address: 0
+# CHECK-NEXT: VM size: 0
+# CHECK-NEXT: File size: 8
+# CHECK-NEXT: Data: ()
+
+# CHECK: Name: .zdebug_info
+# CHECK: regular
+# CHECK: File size: 28
+# CHECK: Data: (
+# CHECK-NEXT: 5A4C4942 00000000 00000008 789C5330 70084828 68980000 09C802C1
+# CHECK-NEXT: )

lldb/test/Shell/lit.site.cfg.py.in

 config.lldb_libs_dir = lit_config.substitute("@LLDB_LIBS_DIR@")
 config.lldb_tools_dir = lit_config.substitute("@LLDB_TOOLS_DIR@")
 # Since it comes from the command line, it may have backslashes which
 # should not need to be escaped.
 config.lldb_lit_tools_dir = lit_config.substitute(r"@LLDB_LIT_TOOLS_DIR@")
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.lldb_enable_lzma = @LLDB_ENABLE_LZMA@
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.lldb_bitness = 64 if @LLDB_IS_64_BITS@ else 32
 config.lldb_enable_python = @LLDB_ENABLE_PYTHON@
 config.lldb_enable_lua = @LLDB_ENABLE_LUA@
 config.lldb_build_directory = "@LLDB_TEST_BUILD_DIRECTORY@"
 config.lldb_system_debugserver = @LLDB_USE_SYSTEM_DEBUGSERVER@
 # The shell tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-shell")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-shell")
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
 
 # Let the main config do the real work.
 lit_config.load_config(config, os.path.join(config.lldb_src_root, "test", "Shell", "lit.cfg.py"))

llvm/include/llvm/Object/Decompressor.h

 //===-- Decompressor.h ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
 #ifndef LLVM_OBJECT_DECOMPRESSOR_H
 #define LLVM_OBJECT_DECOMPRESSOR_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace object {
 
 /// Decompressor helps to handle decompression of compressed sections.
 class Decompressor {
 public:
public:
   Error decompress(MutableArrayRef<uint8_t> Buffer);
 
   /// Return memory buffer size required for decompression.
   uint64_t getDecompressedSize() { return DecompressedSize; }
 
 private:
   Decompressor(StringRef Data);
 
-  Error consumeCompressedZLibHeader(bool Is64Bit, bool IsLittleEndian);
+  Error consumeCompressedSectionHeader(bool Is64Bit, bool IsLittleEndian);
 
   StringRef SectionData;
   uint64_t DecompressedSize;
+  llvm::DebugCompressionType CompressionType = llvm::DebugCompressionType::None;
 };
 
 } // end namespace object
 } // end namespace llvm
 
 #endif // LLVM_OBJECT_DECOMPRESSOR_H

llvm/include/llvm/Support/Compression.h

void compress(Params P, ArrayRef<uint8_t> Input,
               SmallVectorImpl<uint8_t> &Output);
 
 // Decompress Input. The uncompressed size must be available.
 Error decompress(Format F, ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &Output, size_t UncompressedSize);
 Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &Output, size_t UncompressedSize);
 
+Error decompress(Format F, ArrayRef<uint8_t> Input, uint8_t *Output,
+                 size_t &UncompressedSize);
+Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
+                 uint8_t *Output, size_t &UncompressedSize);
+
 } // End of namespace compression
 
 } // End of namespace llvm
 
 #endif

llvm/lib/Object/Decompressor.cpp

 //===-- Decompressor.cpp --------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Object/Decompressor.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 using namespace llvm::support::endian;
 using namespace object;
 
 Expected<Decompressor> Decompressor::create(StringRef Name, StringRef Data,
                                             bool IsLE, bool Is64Bit) {
-  if (!compression::zlib::isAvailable())
-    return createError("zlib is not available");
 
   Decompressor D(Data);
-  if (Error Err = D.consumeCompressedZLibHeader(Is64Bit, IsLE))
+  if (Error Err = D.consumeCompressedSectionHeader(Is64Bit, IsLE))
     return std::move(Err);
   return D;
 }
 
 Decompressor::Decompressor(StringRef Data)
     : SectionData(Data), DecompressedSize(0) {}
 
-Error Decompressor::consumeCompressedZLibHeader(bool Is64Bit,
+Error Decompressor::consumeCompressedSectionHeader(bool Is64Bit,
                                                    bool IsLittleEndian) {
   using namespace ELF;
   uint64_t HdrSize = Is64Bit ? sizeof(Elf64_Chdr) : sizeof(Elf32_Chdr);
   if (SectionData.size() < HdrSize)
     return createError("corrupted compressed section header");
 
   DataExtractor Extractor(SectionData, IsLittleEndian, 0);
   uint64_t Offset = 0;
-  if (Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
-                                             : sizeof(Elf32_Word)) !=
-      ELFCOMPRESS_ZLIB)
+  uint64_t ELFCompressionSchemeId = Extractor.getUnsigned(
+      &Offset, Is64Bit ? sizeof(Elf64_Word) : sizeof(Elf32_Word));
+
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZLIB)
+    CompressionType = llvm::DebugCompressionType::Z;
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZSTD)
+    CompressionType = llvm::DebugCompressionType::Zstd;
+
+  if (CompressionType == llvm::DebugCompressionType::None)
     return createError("unsupported compression type");
+  if (llvm::compression::getReasonIfUnsupported(
+          compression::formatFor(CompressionType)) != nullptr) {
+    switch (compression::formatFor(CompressionType)) {
+    case llvm::compression::Format::Zstd:
+      return createError("zstd is not available");
+    default:
+      return createError("zlib is not available");
+    }
+  }
 
   // Skip Elf64_Chdr::ch_reserved field.
   if (Is64Bit)
     Offset += sizeof(Elf64_Word);
 
   DecompressedSize = Extractor.getUnsigned(
       &Offset, Is64Bit ? sizeof(Elf64_Xword) : sizeof(Elf32_Word));
   SectionData = SectionData.substr(HdrSize);
   return Error::success();
 }
 
 Error Decompressor::decompress(MutableArrayRef<uint8_t> Buffer) {
   size_t Size = Buffer.size();
-  return compression::zlib::uncompress(arrayRefFromStringRef(SectionData),
-                                       Buffer.data(), Size);
+  return compression::decompress(compression::formatFor(CompressionType),
+                                 arrayRefFromStringRef(SectionData),
 }

llvm/lib/Support/Compression.cpp

 }
 
 Error compression::decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
                               SmallVectorImpl<uint8_t> &Output,
                               size_t UncompressedSize) {
   return decompress(formatFor(T), Input, Output, UncompressedSize);
 }
 
+Error compression::decompress(compression::Format F, ArrayRef<uint8_t> Input,
+                              uint8_t *Output, size_t &UncompressedSize) {
+  switch (F) {
+  case compression::Format::Zlib:
+    return zlib::uncompress(Input, Output, UncompressedSize);
+  case compression::Format::Zstd:
+    return zstd::uncompress(Input, Output, UncompressedSize);
+  }
+  llvm_unreachable("");
+}
+
+Error compression::decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
+                              uint8_t *Output, size_t &UncompressedSize) {
+  return decompress(formatFor(T), Input, Output, UncompressedSize);
+}
+
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
   case Z_MEM_ERROR:
     return "zlib error: Z_MEM_ERROR";
   case Z_BUF_ERROR:
     return "zlib error: Z_BUF_ERROR";