[llvm] add zstd to `llvm::compression` namespace

[ckissane]

Summary

• add zstd to llvm::compression namespace
• add a CMake option LLVM_ENABLE_ZSTD with behavior mirroring that of LLVM_ENABLE_ZLIB
• add tests for zstd to llvm/unittests/Support/CompressionTest.cpp
• add a cmake macro (TryFindDependencyMacro.cmake) to aid in finding "modern cmake" style dependencies such as Zstd in an "optional" or "REQUIRED" manner.

[ckissane]

• add_library Zstd::zstd in FindZSTD.cmake

[ckissane]

• simplify zstd cmake code (by using "modern cmake" and no need for FindZSTD anymore)

[ckissane]

• make llvm-config --system-libs support output reflect zstd usage

[ckissane]

• remove get_library_name zstd because non standard

[sebastian-ne]

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

[ckissane]

In D129786#3654396, @sebastian-ne wrote:

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

@sebastian-n yes because facebook is surprisingly good at "modern cmake" practices with shipping libzstd and so `find_dependency(zstd)` is enough to have the lib `zstd::libzstd_shared``` get defined!
I have hopes that this will fix the issues mac os users reported with the old version of this patch.

[ckissane]

make it try to find dependency instead of normal find_dependency macro (because the return breaks things on failure)

[sebastian-ne]

In D129786#3655910, @ckissane wrote:

In D129786#3654396, @sebastian-ne wrote:

It seems like FindZSTD.cmake went missing in the Diff 3 revision of this patch?

@sebastian-n yes because facebook is surprisingly good at "modern cmake" practices with shipping libzstd and so `find_dependency(zstd)` is enough to have the lib `zstd::libzstd_shared``` get defined!
I have hopes that this will fix the issues mac os users reported with the old version of this patch.

Nice, makes sense. It definitely fixes the issue when llvm is included with add_subdirectory.

llvm/cmake/modules/LLVMConfig.cmake.in

81

Should this be lower case zstd? I saw some cmake errors here and they disappear when this is lower case.

[ckissane]

• move try_find_dependency into it's own file

[ckissane]

llvm/cmake/modules/LLVMConfig.cmake.in

81

good catch, It should also probably be another find_dependency call instead of find_package so I fixed that too

[ckissane]

• add newline to end of TryFindDependencyMacro.cmake

[ckissane]

• clarify comment in TryFindDependencyMacro.cmake

[ckissane]

Abandoned because it duplicates https://reviews.llvm.org/D128465

llvm/CMakeLists.txt

 if (CMAKE_SYSTEM_NAME MATCHES "OS390")
   option(LLVM_ENABLE_THREADS "Use threads if available." OFF)
 else()
   option(LLVM_ENABLE_THREADS "Use threads if available." ON)
 endif()
 
 set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
 
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
 set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
 
 set(LLVM_ENABLE_HTTPLIB "OFF" CACHE STRING "Use cpp-httplib HTTP server library if available. Can be ON, OFF, or FORCE_ON")
 
 set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
 
 option(LLVM_ENABLE_Z3_SOLVER
   "Enable Support for the Z3 constraint solver in LLVM."

llvm/cmake/config-ix.cmake

 if( WIN32 AND NOT CYGWIN )
   # We consider Cygwin as another Unix
   set(PURE_WINDOWS 1)
 endif()
 
 include(CheckIncludeFile)
 include(CheckLibraryExists)
 include(CheckSymbolExists)
 include(CheckCXXSymbolExists)
 include(CheckFunctionExists)
 include(CheckStructHasMember)
 include(CheckCCompilerFlag)
 include(CMakePushCheckState)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_DIR}/modules")
+message(INFO ${CMAKE_MODULE_PATH})
+include(TryFindDependencyMacro)
 
 include(CheckCompilerVersion)
 include(CheckProblematicConfigurations)
 include(HandleLLVMStdlib)
 
 if( UNIX AND NOT (APPLE OR BEOS OR HAIKU) )
   # Used by check_symbol_exists:
   list(APPEND CMAKE_REQUIRED_LIBRARIES "m")
if(LLVM_ENABLE_ZLIB STREQUAL FORCE_ON AND NOT HAVE_ZLIB)
       message(FATAL_ERROR "Failed to configure zlib")
     endif()
   endif()
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 else()
   set(LLVM_ENABLE_ZLIB 0)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+    try_find_dependency(zstd REQUIRED)
+    if(NOT zstd_FOUND)
+      message(FATAL_ERROR "Failed to configure zstd, but LLVM_ENABLE_ZSTD is FORCE_ON")
+    endif()
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+    try_find_dependency(zstd)
+  endif()
+  set(LLVM_ENABLE_ZSTD ${zstd_FOUND})
+else()
+  set(LLVM_ENABLE_ZSTD 0)
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
     find_package(LibXml2 REQUIRED)
   elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
     find_package(LibXml2)
   endif()
   if(LibXml2_FOUND)
     # Check if libxml2 we found is usable; for example, we may have found a 32-bit

llvm/cmake/modules/LLVMConfig.cmake.in

 # This file provides information and services to the final user.
 
 @LLVM_CONFIG_CODE@
 
 # For finding self-installed Find*.cmake packages.
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
 
+include(TryFindDependencyMacro)
+
 set(LLVM_VERSION_MAJOR @LLVM_VERSION_MAJOR@)
 set(LLVM_VERSION_MINOR @LLVM_VERSION_MINOR@)
 set(LLVM_VERSION_PATCH @LLVM_VERSION_PATCH@)
 set(LLVM_VERSION_SUFFIX @LLVM_VERSION_SUFFIX@)
 set(LLVM_PACKAGE_VERSION @PACKAGE_VERSION@)
 set(LLVM_PACKAGE_BUGREPORT @PACKAGE_BUGREPORT@)
 
 set(LLVM_BUILD_TYPE @CMAKE_BUILD_TYPE@)

 set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@)
 
 set(LLVM_ENABLE_ZLIB @LLVM_ENABLE_ZLIB@)
 if(LLVM_ENABLE_ZLIB)
   set(ZLIB_ROOT @ZLIB_ROOT@)
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(zstd_ROOT @zstd_ROOT@)
+  try_find_dependency(zstd)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
 endif()
 
 set(LLVM_WITH_Z3 @LLVM_WITH_Z3@)
 
 set(LLVM_ENABLE_DIA_SDK @LLVM_ENABLE_DIA_SDK@)

llvm/cmake/modules/TryFindDependencyMacro.cmake

+# CMakeFindDependencyMacro but with no return call when (NOT ${dep}_FOUND)
+macro(try_find_dependency dep)
+  set(cmake_fd_quiet_arg)
+  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+    set(cmake_fd_quiet_arg QUIET)
+  endif()
+  set(cmake_fd_required_arg)
+  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+    set(cmake_fd_required_arg REQUIRED)
+  endif()
+
+  get_property(cmake_fd_alreadyTransitive GLOBAL PROPERTY
+    _CMAKE_${dep}_TRANSITIVE_DEPENDENCY
+  )
+
+  find_package(${dep} ${ARGN}
+    ${cmake_fd_quiet_arg}
+    ${cmake_fd_required_arg}
+  )
+
+  if(NOT DEFINED cmake_fd_alreadyTransitive OR cmake_fd_alreadyTransitive)
+    set_property(GLOBAL PROPERTY _CMAKE_${dep}_TRANSITIVE_DEPENDENCY TRUE)
+  endif()
+
+  if (NOT ${dep}_FOUND)
+    set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${CMAKE_FIND_PACKAGE_NAME} could not be found because dependency ${dep} could not be found.")
+    set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False)
+  endif()
+  set(cmake_fd_required_arg)
+  set(cmake_fd_quiet_arg)
+endmacro()

llvm/include/llvm/Config/llvm-config.h.cmake

 #cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
 
 /* Define if we have cpp-httplib and want to use it */
 #cmakedefine LLVM_ENABLE_HTTPLIB ${LLVM_ENABLE_HTTPLIB}
 
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
 /* Define to 1 if you have the <sysexits.h> header file. */
 #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
 
 /* Define if the xar_open() function is supported on this platform. */
 #cmakedefine LLVM_HAVE_LIBXAR ${LLVM_HAVE_LIBXAR}

llvm/include/llvm/Support/Compression.h

Error uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
                  size_t &UncompressedSize);
 
 Error uncompress(ArrayRef<uint8_t> Input,
                  SmallVectorImpl<uint8_t> &UncompressedBuffer,
                  size_t UncompressedSize);
 
 } // End of namespace zlib
 
+namespace zstd {
+
+constexpr int NoCompression = -5;
+constexpr int BestSpeedCompression = 1;
+constexpr int DefaultCompression = 5;
+constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+void compress(ArrayRef<uint8_t> Input,
+              SmallVectorImpl<uint8_t> &CompressedBuffer,
+              int Level = DefaultCompression);
+
+Error uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                 size_t &UncompressedSize);
+
+Error uncompress(ArrayRef<uint8_t> Input,
+                 SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                 size_t UncompressedSize);
+
+} // End of namespace zstd
+
 } // End of namespace compression
 
 } // End of namespace llvm
 
 #endif

llvm/lib/Support/CMakeLists.txt

if (NOT LLVM_HAS_NOGLOBAL_CTOR_MUTEX)
     string(REPLACE "-Werror=global-constructors" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
   endif()
 endif()
 
 if(LLVM_ENABLE_ZLIB)
   set(imported_libs ZLIB::ZLIB)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  list(APPEND imported_libs zstd::libzstd_shared)
+endif()
+
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
   set(system_libs ${system_libs} psapi shell32 ole32 uuid advapi32)
 elseif( CMAKE_HOST_UNIX )
   if( HAVE_LIBRT )
     set(system_libs ${system_libs} rt)
   endif()
if(LLVM_ENABLE_ZLIB)
   endif()
   if(NOT zlib_library)
     get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION)
   endif()
   get_library_name(${zlib_library} zlib_library)
   set(llvm_system_libs ${llvm_system_libs} "${zlib_library}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators.
+  if(CMAKE_BUILD_TYPE)
+    string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
+    get_property(zstd_library TARGET zstd::libzstd_shared PROPERTY LOCATION_${build_type})
+  endif()
+  if(NOT zstd_library)
+    get_property(zstd_library TARGET zstd::libzstd_shared PROPERTY LOCATION)
+  endif()
+  set(llvm_system_libs ${llvm_system_libs} "${zstd_library}")
+endif()
+
 if(LLVM_ENABLE_TERMINFO)
   if(NOT terminfo_library)
     get_property(terminfo_library TARGET Terminfo::terminfo PROPERTY LOCATION)
   endif()
   get_library_name(${terminfo_library} terminfo_library)
   set(llvm_system_libs ${llvm_system_libs} "${terminfo_library}")
 endif()
 

llvm/lib/Support/Compression.cpp

 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #if LLVM_ENABLE_ZLIB
 #include <zlib.h>
 #endif
+#if LLVM_ENABLE_ZSTD
+#include <zstd.h>
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
 
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
   switch (Code) {
Error zlib::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 Error zlib::uncompress(ArrayRef<uint8_t> Input,
                        SmallVectorImpl<uint8_t> &UncompressedBuffer,
                        size_t UncompressedSize) {
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef<uint8_t> Input,
+                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+      ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+                      (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+    report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+    CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                       size_t &UncompressedSize) {
+  const size_t Res =
+      ::ZSTD_decompress(UncompressedBuffer, UncompressedSize,
+                        (const uint8_t *)Input.data(), Input.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
+                                                     inconvertibleErrorCode())
+                           : Error::success();
+}
+
+Error zstd::uncompress(ArrayRef<uint8_t> Input,
+                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                       size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E =
+      zstd::uncompress(Input, UncompressedBuffer.data(), UncompressedSize);
+  if (UncompressedSize < UncompressedBuffer.size())
+    UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(ArrayRef<uint8_t> Input,
+                    SmallVectorImpl<uint8_t> &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(ArrayRef<uint8_t> Input, uint8_t *UncompressedBuffer,
+                       size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(ArrayRef<uint8_t> Input,
+                       SmallVectorImpl<uint8_t> &UncompressedBuffer,
+                       size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif

llvm/test/lit.site.cfg.py.in

 config.host_os = "@HOST_OS@"
 config.host_cc = "@HOST_CC@"
 config.host_cxx = "@HOST_CXX@"
 # Note: ldflags can contain double-quoted paths, so must use single quotes here.
 config.host_ldflags = '@HOST_LDFLAGS@'
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
 config.have_httplib = @LLVM_ENABLE_HTTPLIB@
 config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
 config.enable_ffi = @LLVM_ENABLE_FFI@
 config.build_examples = @LLVM_BUILD_EXAMPLES@
 config.enable_threads = @LLVM_ENABLE_THREADS@

llvm/unittests/Support/CompressionTest.cpp

TEST(CompressionTest, Zlib) {
 
   testZlibCompression(BinaryDataStr, zlib::NoCompression);
   testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
   testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
   testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector<uint8_t, 0> Compressed;
+  SmallVector<uint8_t, 0> Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+    // Uncompression fails if expected length is too short.
+    E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+    EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+    BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }

utils/bazel/llvm_configs/llvm-config.h.cmake

 #cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
 
 /* Define if we have cpp-httplib and want to use it */
 #cmakedefine LLVM_ENABLE_HTTPLIB ${LLVM_ENABLE_HTTPLIB}
 
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
 /* Define to 1 if you have the <sysexits.h> header file. */
 #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
 
 /* Define if the xar_open() function is supported on this platform. */
 #cmakedefine LLVM_HAVE_LIBXAR ${LLVM_HAVE_LIBXAR}