[Object] Add zstd decompression support to Decompressor

[MaskRay]

Summary

llvm::object::Decompressor is used by many DWARF consumers like llvm-dwarfdump,
llvm-dwp, llvm-symbolizer. Add tests to them. The lldb test can be left to
D133530.

[dblaikie]

(please include me in review of compression/decompression related stuff in LLVM - as the original author of the functionality in LLVM & someone who's reviewed/been involved in most of the changes since)

[dblaikie]

Looks good to me - thanks for the extra test coverage! (though I'm also OK without it - I figure we don't have to test generic functionality in every user of it (like we wouldn't test every symbolizing feature in both llvm-symbolizer and llvm-addr2line, for instance) - but it's lightweight/easy enough, I guess)

[MaskRay]

In D134116#3797958, @dblaikie wrote:

(please include me in review of compression/decompression related stuff in LLVM - as the original author of the functionality in LLVM & someone who's reviewed/been involved in most of the changes since)

Ah, sorry:( I just copied the original reviewer list but I do know you should be listed.

llvm/include/llvm/Object/Decompressor.h

 //===-- Decompressor.h ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
 #ifndef LLVM_OBJECT_DECOMPRESSOR_H
 #define LLVM_OBJECT_DECOMPRESSOR_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace object {
 
 /// Decompressor helps to handle decompression of compressed sections.
 class Decompressor {
 public:
   /// Create decompressor object.
   /// @param Name        Section name.
   /// @param Data        Section content.
   /// @param IsLE        Flag determines if Data is in little endian form.
   /// @param Is64Bit     Flag determines if object is 64 bit.
   static Expected<Decompressor> create(StringRef Name, StringRef Data,
                                        bool IsLE, bool Is64Bit);
 
   /// Resize the buffer and uncompress section data into it.
   /// @param Out         Destination buffer.
   template <class T> Error resizeAndDecompress(T &Out) {
     Out.resize(DecompressedSize);
     return decompress({(uint8_t *)Out.data(), (size_t)DecompressedSize});
   }
 
   /// Uncompress section data to raw buffer provided.
-  /// @param Buffer      Destination buffer.
-  Error decompress(MutableArrayRef<uint8_t> Buffer);
+  Error decompress(MutableArrayRef<uint8_t> Output);
 
   /// Return memory buffer size required for decompression.
   uint64_t getDecompressedSize() { return DecompressedSize; }
 
 private:
   Decompressor(StringRef Data);
 
-  Error consumeCompressedZLibHeader(bool Is64Bit, bool IsLittleEndian);
+  Error consumeCompressedHeader(bool Is64Bit, bool IsLittleEndian);
 
   StringRef SectionData;
   uint64_t DecompressedSize;
+  DebugCompressionType CompressionType = DebugCompressionType::None;
 };
 
 } // end namespace object
 } // end namespace llvm
 
 #endif // LLVM_OBJECT_DECOMPRESSOR_H

llvm/include/llvm/Support/Compression.h

 const char *getReasonIfUnsupported(Format F);
 
 // Compress Input with the specified format P.Format. If Level is -1, use
 // *::DefaultCompression for the format.
 void compress(Params P, ArrayRef<uint8_t> Input,
               SmallVectorImpl<uint8_t> &Output);
 
 // Decompress Input. The uncompressed size must be available.
+Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
+                 uint8_t *Output, size_t UncompressedSize);
 Error decompress(Format F, ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &Output, size_t UncompressedSize);
 Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &Output, size_t UncompressedSize);
 
 } // End of namespace compression
 
 } // End of namespace llvm
 
 #endif

llvm/lib/Object/Decompressor.cpp

 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::support::endian;
 using namespace object;
 
 Expected<Decompressor> Decompressor::create(StringRef Name, StringRef Data,
                                             bool IsLE, bool Is64Bit) {
-  if (!compression::zlib::isAvailable())
-    return createError("zlib is not available");
-
   Decompressor D(Data);
-  if (Error Err = D.consumeCompressedZLibHeader(Is64Bit, IsLE))
+  if (Error Err = D.consumeCompressedHeader(Is64Bit, IsLE))
     return std::move(Err);
   return D;
 }
 
 Decompressor::Decompressor(StringRef Data)
     : SectionData(Data), DecompressedSize(0) {}
 
-Error Decompressor::consumeCompressedZLibHeader(bool Is64Bit,
-                                                bool IsLittleEndian) {
+Error Decompressor::consumeCompressedHeader(bool Is64Bit, bool IsLittleEndian) {
   using namespace ELF;
   uint64_t HdrSize = Is64Bit ? sizeof(Elf64_Chdr) : sizeof(Elf32_Chdr);
   if (SectionData.size() < HdrSize)
     return createError("corrupted compressed section header");
 
   DataExtractor Extractor(SectionData, IsLittleEndian, 0);
   uint64_t Offset = 0;
-  if (Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
-                                             : sizeof(Elf32_Word)) !=
-      ELFCOMPRESS_ZLIB)
-    return createError("unsupported compression type");
+  auto ChType = Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
+                                                       : sizeof(Elf32_Word));
+  switch (ChType) {
+  case ELFCOMPRESS_ZLIB:
+    CompressionType = DebugCompressionType::Zlib;
+    break;
+  case ELFCOMPRESS_ZSTD:
+    CompressionType = DebugCompressionType::Zstd;
+    break;
+  default:
+    return createError("unsupported compression type (" + Twine(ChType) + ")");
+  }
+  if (const char *Reason = llvm::compression::getReasonIfUnsupported(
+          compression::formatFor(CompressionType)))
+    return createError(Reason);
 
   // Skip Elf64_Chdr::ch_reserved field.
   if (Is64Bit)
     Offset += sizeof(Elf64_Word);
 
   DecompressedSize = Extractor.getUnsigned(
       &Offset, Is64Bit ? sizeof(Elf64_Xword) : sizeof(Elf32_Word));
   SectionData = SectionData.substr(HdrSize);
   return Error::success();
 }
 
-Error Decompressor::decompress(MutableArrayRef<uint8_t> Buffer) {
-  size_t Size = Buffer.size();
-  return compression::zlib::decompress(arrayRefFromStringRef(SectionData),
-                                       Buffer.data(), Size);
+Error Decompressor::decompress(MutableArrayRef<uint8_t> Output) {
+  return compression::decompress(CompressionType,
+                                 arrayRefFromStringRef(SectionData),
+                                 Output.data(), Output.size());
 }

llvm/lib/Support/Compression.cpp

case compression::Format::Zlib:
     zlib::compress(Input, Output, P.level);
     break;
   case compression::Format::Zstd:
     zstd::compress(Input, Output, P.level);
     break;
   }
 }
 
+Error compression::decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
+                              uint8_t *Output, size_t UncompressedSize) {
+  switch (formatFor(T)) {
+  case compression::Format::Zlib:
+    return zlib::decompress(Input, Output, UncompressedSize);
+  case compression::Format::Zstd:
+    return zstd::decompress(Input, Output, UncompressedSize);
+  }
+  llvm_unreachable("");
+}
+
 Error compression::decompress(compression::Format F, ArrayRef<uint8_t> Input,
                               SmallVectorImpl<uint8_t> &Output,
                               size_t UncompressedSize) {
   switch (F) {
   case compression::Format::Zlib:
     return zlib::decompress(Input, Output, UncompressedSize);
   case compression::Format::Zstd:
     return zstd::decompress(Input, Output, UncompressedSize);

llvm/test/DebugInfo/dwarfdump-zstd.test

+; REQUIRES: zstd
+;; llvm-dwarfdump supports zstd compressed debug sections.
+
+; RUN: llvm-objcopy --compress-debug-sections=zstd %S/Inputs/dwarfdump-test.elf-x86-64 %t
+; RUN: llvm-dwarfdump %t | FileCheck %s
+
+; CHECK: .debug_info contents
+; CHECK:   DW_AT_name ("dwarfdump-test.cc")

llvm/test/DebugInfo/symbolize-zstd.test

+# REQUIRES: zstd
+## llvm-symbolizer supports zstd compressed debug sections.
+
+# RUN: llvm-objcopy --compress-debug-sections=zstd %S/Inputs/dwarfdump-test.elf-x86-64 %t
+# RUN: llvm-symbolizer --obj=%t 0x40113f | FileCheck %s
+
+# CHECK:      main
+# CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16

llvm/test/tools/llvm-dwp/X86/compress-zstd.test

+# REQUIRES: zstd
+## llvm-dwp supports zstd compressed debug sections.
+
+# RUN: llvm-objcopy --compress-debug-sections=zstd %p/../Inputs/simple/notypes/a.dwo %t.o
+# RUN: llvm-dwp %t.o -o %t.dwp
+
+# CHECK: .debug_info.dwo contents:
+# CHECK: DW_TAG_compile_unit
+# CHECK:   DW_AT_name {{.*}} "a.cpp"