[llvm] add zstd to `llvm::compression` namespace

[ckissane]

Summary

• add zstd to llvm::compression namespace
• add a CMake option LLVM_ENABLE_ZSTD with behavior mirroring that of LLVM_ENABLE_ZLIB
• add tests for zstd to llvm/unittests/Support/CompressionTest.cpp
• add a cmake macro (TryFindDependencyMacro.cmake) to aid in finding "modern cmake" style dependencies such as Zstd in an "optional" or "REQUIRED" manner.

[ckissane]

• add_library Zstd::zstd in FindZSTD.cmake

[ckissane]

• simplify zstd cmake code (by using "modern cmake" and no need for FindZSTD anymore)

[ckissane]

• make llvm-config --system-libs support output reflect zstd usage

[ckissane]

• remove get_library_name zstd because non standard

[sebastian-ne]

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

[ckissane]

In D129786#3654396, @sebastian-ne wrote:

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

@sebastian-n yes because facebook is surprisingly good at "modern cmake" practices with shipping libzstd and so `find_dependency(zstd)` is enough to have the lib `zstd::libzstd_shared``` get defined!
I have hopes that this will fix the issues mac os users reported with the old version of this patch.

[ckissane]

make it try to find dependency instead of normal find_dependency macro (because the return breaks things on failure)

[sebastian-ne]

In D129786#3655910, @ckissane wrote:

In D129786#3654396, @sebastian-ne wrote:

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

@sebastian-n yes because facebook is surprisingly good at "modern cmake" practices with shipping libzstd and so `find_dependency(zstd)` is enough to have the lib `zstd::libzstd_shared``` get defined!
I have hopes that this will fix the issues mac os users reported with the old version of this patch.

Nice, makes sense. It definitely fixes the issue when llvm is included with add_subdirectory.

llvm/cmake/modules/LLVMConfig.cmake.in

79

Should this be lower case zstd? I saw some cmake errors here and they disappear when this is lower case.

[ckissane]

• move try_find_dependency into it's own file

[ckissane]

llvm/cmake/modules/LLVMConfig.cmake.in

79

good catch, It should also probably be another find_dependency call instead of find_package so I fixed that too

[ckissane]

• add newline to end of TryFindDependencyMacro.cmake

[ckissane]

• clarify comment in TryFindDependencyMacro.cmake

[ckissane]

Abandoned because it duplicates https://reviews.llvm.org/D128465

llvm/CMakeLists.txt

 if (CMAKE_SYSTEM_NAME MATCHES "OS390")
   option(LLVM_ENABLE_THREADS "Use threads if available." OFF)
 else()
   option(LLVM_ENABLE_THREADS "Use threads if available." ON)
 endif()
 
 set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
 
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
 set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
 
 set(LLVM_ENABLE_HTTPLIB "OFF" CACHE STRING "Use cpp-httplib HTTP server library if available. Can be ON, OFF, or FORCE_ON")
 
 set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
 
 option(LLVM_ENABLE_Z3_SOLVER
   "Enable Support for the Z3 constraint solver in LLVM."

llvm/cmake/config-ix.cmake

if(LLVM_ENABLE_ZLIB STREQUAL FORCE_ON AND NOT HAVE_ZLIB)
       message(FATAL_ERROR "Failed to configure zlib")
     endif()
   endif()
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 else()
   set(LLVM_ENABLE_ZLIB 0)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+    find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+    find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+    # Check if zstd we found is usable; for example, we may have found a 32-bit
+    # library on a 64-bit system which would result in a link-time failure.
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+    list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+    check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+    cmake_pop_check_state()
+  else()
+    set(HAVE_ZSTD 0)
+  endif()
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+    message(FATAL_ERROR "Failed to configure zstd")
+  endif()
+  if(HAVE_ZSTD)
+    set(LLVM_ENABLE_ZSTD 1)
+  else()
+    set(LLVM_ENABLE_ZSTD 0)
+  endif()
+else()
+  set(LLVM_ENABLE_ZSTD 0)
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
     find_package(LibXml2 REQUIRED)
   elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
     find_package(LibXml2)
   endif()
   if(LibXml2_FOUND)
     # Check if libxml2 we found is usable; for example, we may have found a 32-bit

llvm/cmake/modules/FindZSTD.cmake

+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    ZSTD DEFAULT_MSG
+    ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+    set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+    set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)

llvm/cmake/modules/LLVMConfig.cmake.in

 set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@)
 
 set(LLVM_ENABLE_ZLIB @LLVM_ENABLE_ZLIB@)
 if(LLVM_ENABLE_ZLIB)
   set(ZLIB_ROOT @ZLIB_ROOT@)
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
 endif()
 
 set(LLVM_WITH_Z3 @LLVM_WITH_Z3@)
 
 set(LLVM_ENABLE_DIA_SDK @LLVM_ENABLE_DIA_SDK@)

llvm/include/llvm/Config/llvm-config.h.cmake

 #cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
 
 /* Define if we have cpp-httplib and want to use it */
 #cmakedefine LLVM_ENABLE_HTTPLIB ${LLVM_ENABLE_HTTPLIB}
 
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
 /* Define to 1 if you have the <sysexits.h> header file. */
 #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
 
 /* Define if the xar_open() function is supported on this platform. */
 #cmakedefine LLVM_HAVE_LIBXAR ${LLVM_HAVE_LIBXAR}

llvm/include/llvm/Support/Compression.h

Error uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
                  size_t &UncompressedSize);
 
 Error uncompress(ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &UncompressedBuffer,
                  size_t UncompressedSize);
 
 } // End of namespace zlib
 
+namespace zstd {
+
+constexpr int NoCompression = -5;
+constexpr int BestSpeedCompression = 1;
+constexpr int DefaultCompression = 5;
+constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+void compress(ArrayRef<uint8_t> Input,
+              SmallVectorImpl<uint8_t> &CompressedBuffer,
+              int Level = DefaultCompression);
+
+Error uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                 size_t &UncompressedSize);
+
+Error uncompress(ArrayRef<uint8_t> Input,
+                 SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                 size_t UncompressedSize);
+
+} // End of namespace zstd
+
 } // End of namespace compression
 
 } // End of namespace llvm
 
 #endif

llvm/lib/Support/CMakeLists.txt

if (NOT LLVM_HAS_NOGLOBAL_CTOR_MUTEX)
     string(REPLACE "-Werror=global-constructors" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
   endif()
 endif()
 
 if(LLVM_ENABLE_ZLIB)
   set(imported_libs ZLIB::ZLIB)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  list(APPEND imported_libs zstd)
+endif()
+
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
   set(system_libs ${system_libs} psapi shell32 ole32 uuid advapi32)
 elseif( CMAKE_HOST_UNIX )
   if( HAVE_LIBRT )
     set(system_libs ${system_libs} rt)
   endif()

llvm/lib/Support/Compression.cpp

 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #if LLVM_ENABLE_ZLIB
 #include <zlib.h>
 #endif
+#if LLVM_ENABLE_ZSTD
+#include <zstd.h>
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
 
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
Error zlib::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 Error zlib::uncompress(ArrayRef<uint8_t> Input,
                        SmallVectorImpl<uint8_t> &UncompressedBuffer,
                        size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef<uint8_t> Input,
+                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+      ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+                      (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+    report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+    CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                       size_t &UncompressedSize) {
+  const size_t Res =
+      ::ZSTD_decompress(UncompressedBuffer, UncompressedSize,
+                        (const uint8_t *)Input.data(), Input.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
+                                                     inconvertibleErrorCode())
+                           : Error::success();
+}
+
+Error zstd::uncompress(ArrayRef<uint8_t> Input,
+                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                       size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E =
+      zstd::uncompress(Input, UncompressedBuffer.data(), UncompressedSize);
+  if (UncompressedSize < UncompressedBuffer.size())
+    UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(ArrayRef<uint8_t> Input,
+                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                       size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(ArrayRef<uint8_t> Input,
+                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                       size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif

llvm/test/lit.site.cfg.py.in

 config.host_os = "@HOST_OS@"
 config.host_cc = "@HOST_CC@"
 config.host_cxx = "@HOST_CXX@"
 # Note: ldflags can contain double-quoted paths, so must use single quotes here.
 config.host_ldflags = '@HOST_LDFLAGS@'
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
 config.have_httplib = @LLVM_ENABLE_HTTPLIB@
 config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
 config.enable_ffi = @LLVM_ENABLE_FFI@
 config.build_examples = @LLVM_BUILD_EXAMPLES@
 config.enable_threads = @LLVM_ENABLE_THREADS@

llvm/unittests/Support/CompressionTest.cpp

TEST(CompressionTest, Zlib) {
 
   testZlibCompression(BinaryDataStr, zlib::NoCompression);
   testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
   testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
   testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector<uint8_t, 0> Compressed;
+  SmallVector<uint8_t, 0> Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+    // Uncompression fails if expected length is too short.
+    E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+    EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+    BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }

utils/bazel/llvm_configs/llvm-config.h.cmake

 #cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
 
 /* Define if we have cpp-httplib and want to use it */
 #cmakedefine LLVM_ENABLE_HTTPLIB ${LLVM_ENABLE_HTTPLIB}
 
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
 /* Define to 1 if you have the <sysexits.h> header file. */
 #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
 
 /* Define if the xar_open() function is supported on this platform. */
 #cmakedefine LLVM_HAVE_LIBXAR ${LLVM_HAVE_LIBXAR}