Alternative compression API design for illustration

[dblaikie]

Summary

An alternative/hybrid between D130516 and D130506

[MaskRay]

The base probably should be set to the current state of llvm/lib/Support/Compression.cpp for comparison, instead of a rebase on D130516

[dblaikie]

In D131638#3714663, @MaskRay wrote:

The base probably should be set to the current state of llvm/lib/Support/Compression.cpp for comparison, instead of a rebase on D130516

Oh, right ,sorry - wasn't my intent that it be based on that - thought that was just an implementation detail of how I'd got here. I'll see about getting that fixed up.

[dblaikie]

rebase on main instead of D130516

[MaskRay]

Is there a demonstrating how this performs better than D130506 with an example like llvm-objcopy --compress-debug-sections=zstd?

[MaskRay]

In D131638#3717705, @MaskRay wrote:

Is there a demonstrating how this performs better than D130506 with an example like llvm-objcopy --compress-debug-sections=zstd?

Ping this question.

[dblaikie]

In D131638#3733745, @MaskRay wrote:

In D131638#3717705, @MaskRay wrote:

Is there a demonstrating how this performs better than D130506 with an example like llvm-objcopy --compress-debug-sections=zstd?

Ping this question.

After various design discussions I think D131992 is, for my design preferences, more well fleshed out than this patch, so I'm abandoning this one in favor of that one. I'll follow up on D131992 with more thoughts there.

llvm/include/llvm/MC/MCTargetOptions.h

 //===- MCTargetOptions.h - MC Target Options --------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_MC_MCTARGETOPTIONS_H
 #define LLVM_MC_MCTARGETOPTIONS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Compression.h"
 #include <string>
 #include <vector>
 
 namespace llvm {
 
 enum class ExceptionHandling {
   None,     ///< No exception support
   DwarfCFI, ///< DWARF-like instruction based exceptions
   SjLj,     ///< setjmp/longjmp based exceptions
   ARM,      ///< ARM EHABI
   WinEH,    ///< Windows Exception Handling
   Wasm,     ///< WebAssembly Exception Handling
   AIX,      ///< AIX Exception Handling
 };
 
-enum class DebugCompressionType {
-  None, ///< No compression
-  Z,    ///< zlib style complession
-};
-
 enum class EmitDwarfUnwindType {
   Always,          // Always emit dwarf unwind
   NoCompactUnwind, // Only emit if compact unwind isn't available
   Default,         // Default behavior is based on the target
 };
 
 class StringRef;
 

llvm/include/llvm/Support/Compression.h

 //===-- llvm/Support/Compression.h ---Compression----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains basic functions for compression/uncompression.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_SUPPORT_COMPRESSION_H
 #define LLVM_SUPPORT_COMPRESSION_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
 template <typename T> class SmallVectorImpl;
 class Error;
 
+enum class DebugCompressionType { None, Z, Zstd };
+
 namespace compression {
+
+struct CompressionAlgorithm {
+  const StringRef Name;
+  const int BestSpeedLevel;
+  const int DefaultLevel;
+  const int BestSizeLevel;
+  const bool Available;
+  virtual void compress(ArrayRef<uint8_t> Input,
+                        SmallVectorImpl<uint8_t> &CompressedBuffer,
+                        int Level) const = 0;
+  virtual Error decompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                           size_t &UncompressedSize) const = 0;
+  void compress(ArrayRef<uint8_t> Input,
+                SmallVectorImpl<uint8_t> &CompressedBuffer) const {
+    return compress(Input, CompressedBuffer, this->DefaultLevel);
+  }
+
+  Error decompress(ArrayRef<uint8_t> Input,
+                   SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                   size_t UncompressedSize) const {
+    UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+    Error E = decompress(Input, UncompressedBuffer.data(), UncompressedSize);
+    if (UncompressedSize < UncompressedBuffer.size())
+      UncompressedBuffer.truncate(UncompressedSize);
+    return E;
+  }
+
+  explicit operator bool() const { return Available; }
+
+protected:
+  CompressionAlgorithm(StringRef Name, int BestSpeedLevel, int DefaultLevel,
+                       int BestSizeLevel, bool Available)
+      : Name(Name), BestSpeedLevel(BestSpeedLevel), DefaultLevel(DefaultLevel),
+        BestSizeLevel(BestSizeLevel), Available(Available) {}
+  ~CompressionAlgorithm() = default;
+};
+
+const CompressionAlgorithm &getCompressionAlgorithm(DebugCompressionType Kind);
+
+// deprecated API
 namespace zlib {
 
 constexpr int NoCompression = 0;
 constexpr int BestSpeedCompression = 1;
 constexpr int DefaultCompression = 6;
 constexpr int BestSizeCompression = 9;
 
 bool isAvailable();

llvm/lib/ObjCopy/ELF/ELFObject.cpp

Error ELFSectionWriter<ELFT>::visit(const CompressedSection &Sec) {
   Elf_Chdr_Impl<ELFT> Chdr = {};
   switch (Sec.CompressionType) {
   case DebugCompressionType::None:
     std::copy(Sec.OriginalData.begin(), Sec.OriginalData.end(), Buf);
     return Error::success();
   case DebugCompressionType::Z:
     Chdr.ch_type = ELF::ELFCOMPRESS_ZLIB;
     break;
+  case DebugCompressionType::Zstd:
+    llvm_unreachable("unsupported");
   }
   Chdr.ch_size = Sec.DecompressedSize;
   Chdr.ch_addralign = Sec.DecompressedAlign;
   memcpy(Buf, &Chdr, sizeof(Chdr));
   Buf += sizeof(Chdr);
 
   std::copy(Sec.CompressedData.begin(), Sec.CompressedData.end(), Buf);
   return Error::success();

llvm/lib/Support/Compression.cpp

 #include "llvm/Config/config.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #if LLVM_ENABLE_ZLIB
 #include <zlib.h>
 #endif
 #if LLVM_ENABLE_ZSTD
-#include <zstd.h>
+#include <zlib.h>
 #endif
 
 using namespace llvm;
 using namespace llvm::compression;
 
+namespace {
+
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
   case Z_MEM_ERROR:
     return "zlib error: Z_MEM_ERROR";
   case Z_BUF_ERROR:
     return "zlib error: Z_BUF_ERROR";
   case Z_STREAM_ERROR:
     return "zlib error: Z_STREAM_ERROR";
   case Z_DATA_ERROR:
     return "zlib error: Z_DATA_ERROR";
   case Z_OK:
   default:
     llvm_unreachable("unknown or unexpected zlib status code");
   }
 }
-
-bool zlib::isAvailable() { return true; }
-
-void zlib::compress(ArrayRef<uint8_t> Input,
-                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+#endif
+namespace {
+struct ZlibCompressionAlgorithm final : public CompressionAlgorithm {
+#if LLVM_ENABLE_ZLIB
+  void compress(ArrayRef<uint8_t> Input,
+                SmallVectorImpl<uint8_t> &CompressedBuffer,
+                int Level) const override {
     unsigned long CompressedSize = ::compressBound(Input.size());
     CompressedBuffer.resize_for_overwrite(CompressedSize);
     int Res = ::compress2((Bytef *)CompressedBuffer.data(), &CompressedSize,
                           (const Bytef *)Input.data(), Input.size(), Level);
     if (Res == Z_MEM_ERROR)
       report_bad_alloc_error("Allocation failed");
     assert(Res == Z_OK);
     // Tell MemorySanitizer that zlib output buffer is fully initialized.
     // This avoids a false report when running LLVM with uninstrumented ZLib.
     __msan_unpoison(CompressedBuffer.data(), CompressedSize);
     if (CompressedSize < CompressedBuffer.size())
       CompressedBuffer.truncate(CompressedSize);
-}
-
-Error zlib::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
-                       size_t &UncompressedSize) {
+  };
+  Error decompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                   size_t &UncompressedSize) const override {
     int Res =
         ::uncompress((Bytef *)UncompressedBuffer, (uLongf *)&UncompressedSize,
                      (const Bytef *)Input.data(), Input.size());
     // Tell MemorySanitizer that zlib output buffer is fully initialized.
     // This avoids a false report when running LLVM with uninstrumented ZLib.
     __msan_unpoison(UncompressedBuffer, UncompressedSize);
     return Res ? make_error<StringError>(convertZlibCodeToString(Res),
                                          inconvertibleErrorCode())
                : Error::success();
-}
-
-Error zlib::uncompress(ArrayRef<uint8_t> Input,
-                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
-                       size_t UncompressedSize) {
-  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
-  Error E =
-      zlib::uncompress(Input, UncompressedBuffer.data(), UncompressedSize);
-  if (UncompressedSize < UncompressedBuffer.size())
-    UncompressedBuffer.truncate(UncompressedSize);
-  return E;
-}
-
+  };
 #else
-bool zlib::isAvailable() { return false; }
-void zlib::compress(ArrayRef<uint8_t> Input,
-                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
-  llvm_unreachable("zlib::compress is unavailable");
-}
-Error zlib::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
-                       size_t &UncompressedSize) {
-  llvm_unreachable("zlib::uncompress is unavailable");
-}
-Error zlib::uncompress(ArrayRef<uint8_t> Input,
-                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
-                       size_t UncompressedSize) {
-  llvm_unreachable("zlib::uncompress is unavailable");
-}
+  void compress(ArrayRef<uint8_t> Input,
+                SmallVectorImpl<uint8_t> &CompressedBuffer,
+                int Level) const override {
+    llvm_unreachable("method:\"compress\" is unsupported for compression "
+                     "algorithm:\"zlib\", "
+                     "reason:\"llvm not compiled with zlib support\"");
+  };
+  Error decompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                   size_t &UncompressedSize) const override {
+    llvm_unreachable(
+        "method:\"decompress\" is unsupported for compression "
+        "algorithm:\"zlib\", reason:\"llvm not compiled with zlib support\"");
+  };
 #endif
+  ZlibCompressionAlgorithm()
+      : CompressionAlgorithm("zlib", 1, 6, 9, LLVM_ENABLE_ZLIB) {}
+};
 
+struct ZStdCompressionAlgorithm final : public CompressionAlgorithm {
 #if LLVM_ENABLE_ZSTD
-
-bool zstd::isAvailable() { return true; }
-
-void zstd::compress(ArrayRef<uint8_t> Input,
-                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  void compress(ArrayRef<uint8_t> Input,
+                SmallVectorImpl<uint8_t> &CompressedBuffer,
+                int Level) const override {
     unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
     CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
     unsigned long CompressedSize =
         ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
                         (const char *)Input.data(), Input.size(), Level);
     if (ZSTD_isError(CompressedSize))
       report_bad_alloc_error("Allocation failed");
-  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+    // Tell MemorySanitizer that zlib output buffer is fully initialized.
     // This avoids a false report when running LLVM with uninstrumented ZLib.
     __msan_unpoison(CompressedBuffer.data(), CompressedSize);
     if (CompressedSize < CompressedBuffer.size())
       CompressedBuffer.truncate(CompressedSize);
-}
-
-Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
-                       size_t &UncompressedSize) {
+  };
+  Error decompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                   size_t &UncompressedSize) const override {
     const size_t Res =
         ::ZSTD_decompress(UncompressedBuffer, UncompressedSize,
                           (const uint8_t *)Input.data(), Input.size());
     UncompressedSize = Res;
-  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+    // Tell MemorySanitizer that zlib output buffer is fully initialized.
     // This avoids a false report when running LLVM with uninstrumented ZLib.
     __msan_unpoison(UncompressedBuffer, UncompressedSize);
     return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
                                                        inconvertibleErrorCode())
                              : Error::success();
+  };
+#else
+  void compress(ArrayRef<uint8_t> Input,
+                SmallVectorImpl<uint8_t> &CompressedBuffer,
+                int Level) const override {
+    llvm_unreachable("zstd compression unsupported");
+  }
+  Error decompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                   size_t &UncompressedSize) const override {
+    llvm_unreachable("zstd decompression unsupported");
   }
+#endif
 
-Error zstd::uncompress(ArrayRef<uint8_t> Input,
-                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
-                       size_t UncompressedSize) {
-  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
-  Error E =
-      zstd::uncompress(Input, UncompressedBuffer.data(), UncompressedSize);
-  if (UncompressedSize < UncompressedBuffer.size())
-    UncompressedBuffer.truncate(UncompressedSize);
-  return E;
+  ZStdCompressionAlgorithm()
+      : CompressionAlgorithm("zlib", 1, 5, 12, LLVM_ENABLE_ZSTD) {}
+};
+} // namespace
+
+} // namespace
+
+const CompressionAlgorithm &
+compression::getCompressionAlgorithm(DebugCompressionType Kind) {
+  switch (Kind) {
+  case DebugCompressionType::Z:
+    static ZlibCompressionAlgorithm ZlibI;
+    return ZlibI;
+  case DebugCompressionType::Zstd:
+    static ZStdCompressionAlgorithm ZStdI;
+    return ZStdI;
+  case DebugCompressionType::None:
+    llvm_unreachable("Only query for actual compression types.");
+  }
+  llvm_unreachable("Unknown compression kind");
+}
+
+// Deprecated implementation
+bool zstd::isAvailable() {
+  return !!getCompressionAlgorithm(DebugCompressionType::Zstd);
 }
-
-#else
-bool zstd::isAvailable() { return false; }
 void zstd::compress(ArrayRef<uint8_t> Input,
                     SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
-  llvm_unreachable("zstd::compress is unavailable");
+  getCompressionAlgorithm(DebugCompressionType::Zstd)
+      .compress(Input, CompressedBuffer, Level);
 }
 Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
                        size_t &UncompressedSize) {
-  llvm_unreachable("zstd::uncompress is unavailable");
+  return getCompressionAlgorithm(DebugCompressionType::Zstd)
+      .decompress(Input, UncompressedBuffer, UncompressedSize);
 }
 Error zstd::uncompress(ArrayRef<uint8_t> Input,
                        SmallVectorImpl<uint8_t> &UncompressedBuffer,
                        size_t UncompressedSize) {
-  llvm_unreachable("zstd::uncompress is unavailable");
+  return getCompressionAlgorithm(DebugCompressionType::Zstd)
+      .decompress(Input, UncompressedBuffer, UncompressedSize);
+}
+bool zlib::isAvailable() {
+  return !!getCompressionAlgorithm(DebugCompressionType::Z);
+}
+void zlib::compress(ArrayRef<uint8_t> Input,
+                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  getCompressionAlgorithm(DebugCompressionType::Z)
+      .compress(Input, CompressedBuffer, Level);
+}
+Error zlib::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                       size_t &UncompressedSize) {
+  return getCompressionAlgorithm(DebugCompressionType::Z)
+      .decompress(Input, UncompressedBuffer, UncompressedSize);
+}
+Error zlib::uncompress(ArrayRef<uint8_t> Input,
+                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                       size_t UncompressedSize) {
+  return getCompressionAlgorithm(DebugCompressionType::Z)
+      .decompress(Input, UncompressedBuffer, UncompressedSize);
 }
-#endif