diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -438,6 +438,8 @@ set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON") +set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON") + set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON") set(LLVM_ENABLE_HTTPLIB "OFF" CACHE STRING "Use cpp-httplib HTTP server library if available. Can be ON, OFF, or FORCE_ON") diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake --- a/llvm/cmake/config-ix.cmake +++ b/llvm/cmake/config-ix.cmake @@ -11,6 +11,9 @@ include(CheckStructHasMember) include(CheckCCompilerFlag) include(CMakePushCheckState) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_DIR}/modules") +message(INFO ${CMAKE_MODULE_PATH}) +include(TryFindDependencyMacro) include(CheckCompilerVersion) include(CheckProblematicConfigurations) @@ -138,6 +141,20 @@ set(LLVM_ENABLE_ZLIB 0) endif() +if(LLVM_ENABLE_ZSTD) + if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON) + try_find_dependency(zstd REQUIRED) + if(NOT zstd_FOUND) + message(FATAL_ERROR "Failed to configure zstd, but LLVM_ENABLE_ZSTD is FORCE_ON") + endif() + elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*") + try_find_dependency(zstd) + endif() + set(LLVM_ENABLE_ZSTD ${zstd_FOUND}) +else() + set(LLVM_ENABLE_ZSTD 0) +endif() + if(LLVM_ENABLE_LIBXML2) if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON) find_package(LibXml2 REQUIRED) diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in --- a/llvm/cmake/modules/LLVMConfig.cmake.in +++ b/llvm/cmake/modules/LLVMConfig.cmake.in @@ -5,6 +5,8 @@ # For finding self-installed Find*.cmake packages. list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") +include(TryFindDependencyMacro) + set(LLVM_VERSION_MAJOR @LLVM_VERSION_MAJOR@) set(LLVM_VERSION_MINOR @LLVM_VERSION_MINOR@) set(LLVM_VERSION_PATCH @LLVM_VERSION_PATCH@) @@ -73,6 +75,12 @@ find_package(ZLIB) endif() +set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@) +if(LLVM_ENABLE_ZSTD) + set(zstd_ROOT @zstd_ROOT@) + try_find_dependency(zstd) +endif() + set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@) if(LLVM_ENABLE_LIBXML2) find_package(LibXml2) diff --git a/llvm/cmake/modules/TryFindDependencyMacro.cmake b/llvm/cmake/modules/TryFindDependencyMacro.cmake new file mode 100644 --- /dev/null +++ b/llvm/cmake/modules/TryFindDependencyMacro.cmake @@ -0,0 +1,31 @@ +# CMakeFindDependencyMacro but with no return call when (NOT ${dep}_FOUND) +macro(try_find_dependency dep) + set(cmake_fd_quiet_arg) + if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY) + set(cmake_fd_quiet_arg QUIET) + endif() + set(cmake_fd_required_arg) + if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED) + set(cmake_fd_required_arg REQUIRED) + endif() + + get_property(cmake_fd_alreadyTransitive GLOBAL PROPERTY + _CMAKE_${dep}_TRANSITIVE_DEPENDENCY + ) + + find_package(${dep} ${ARGN} + ${cmake_fd_quiet_arg} + ${cmake_fd_required_arg} + ) + + if(NOT DEFINED cmake_fd_alreadyTransitive OR cmake_fd_alreadyTransitive) + set_property(GLOBAL PROPERTY _CMAKE_${dep}_TRANSITIVE_DEPENDENCY TRUE) + endif() + + if (NOT ${dep}_FOUND) + set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${CMAKE_FIND_PACKAGE_NAME} could not be found because dependency ${dep} could not be found.") + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False) + endif() + set(cmake_fd_required_arg) + set(cmake_fd_quiet_arg) +endmacro() diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake --- a/llvm/include/llvm/Config/llvm-config.h.cmake +++ b/llvm/include/llvm/Config/llvm-config.h.cmake @@ -95,6 +95,9 @@ /* Define if zlib compression is available */ #cmakedefine01 LLVM_ENABLE_ZLIB +/* Define if zstd compression is available */ +#cmakedefine01 LLVM_ENABLE_ZSTD + /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */ #cmakedefine LLVM_HAVE_TF_API diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -44,6 +44,28 @@ } // End of namespace zlib +namespace zstd { + +constexpr int NoCompression = -5; +constexpr int BestSpeedCompression = 1; +constexpr int DefaultCompression = 5; +constexpr int BestSizeCompression = 12; + +bool isAvailable(); + +void compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression); + +Error uncompress(ArrayRef Input, uint8_t *UncompressedBuffer, + size_t &UncompressedSize); + +Error uncompress(ArrayRef Input, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize); + +} // End of namespace zstd + } // End of namespace compression } // End of namespace llvm diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -25,6 +25,10 @@ set(imported_libs ZLIB::ZLIB) endif() +if(LLVM_ENABLE_ZSTD) + list(APPEND imported_libs zstd::libzstd_shared) +endif() + if( MSVC OR MINGW ) # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc. # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc. @@ -289,6 +293,18 @@ set(llvm_system_libs ${llvm_system_libs} "${zlib_library}") endif() +if(LLVM_ENABLE_ZSTD) + # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators. + if(CMAKE_BUILD_TYPE) + string(TOUPPER ${CMAKE_BUILD_TYPE} build_type) + get_property(zstd_library TARGET zstd::libzstd_shared PROPERTY LOCATION_${build_type}) + endif() + if(NOT zstd_library) + get_property(zstd_library TARGET zstd::libzstd_shared PROPERTY LOCATION) + endif() + set(llvm_system_libs ${llvm_system_libs} "${zstd_library}") +endif() + if(LLVM_ENABLE_TERMINFO) if(NOT terminfo_library) get_property(terminfo_library TARGET Terminfo::terminfo PROPERTY LOCATION) diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -20,6 +20,9 @@ #if LLVM_ENABLE_ZLIB #include #endif +#if LLVM_ENABLE_ZSTD +#include +#endif using namespace llvm; using namespace llvm::compression; @@ -100,3 +103,65 @@ llvm_unreachable("zlib::uncompress is unavailable"); } #endif + +#if LLVM_ENABLE_ZSTD + +bool zstd::isAvailable() { return true; } + +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size()); + CompressedBuffer.resize_for_overwrite(CompressedBufferSize); + unsigned long CompressedSize = + ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize, + (const char *)Input.data(), Input.size(), Level); + if (ZSTD_isError(CompressedSize)) + report_bad_alloc_error("Allocation failed"); + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + if (CompressedSize < CompressedBuffer.size()) + CompressedBuffer.truncate(CompressedSize); +} + +Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer, + size_t &UncompressedSize) { + const size_t Res = + ::ZSTD_decompress(UncompressedBuffer, UncompressedSize, + (const uint8_t *)Input.data(), Input.size()); + UncompressedSize = Res; + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(UncompressedBuffer, UncompressedSize); + return ZSTD_isError(Res) ? make_error(ZSTD_getErrorName(Res), + inconvertibleErrorCode()) + : Error::success(); +} + +Error zstd::uncompress(ArrayRef Input, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + UncompressedBuffer.resize_for_overwrite(UncompressedSize); + Error E = + zstd::uncompress(Input, UncompressedBuffer.data(), UncompressedSize); + if (UncompressedSize < UncompressedBuffer.size()) + UncompressedBuffer.truncate(UncompressedSize); + return E; +} + +#else +bool zstd::isAvailable() { return false; } +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + llvm_unreachable("zstd::compress is unavailable"); +} +Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer, + size_t &UncompressedSize) { + llvm_unreachable("zstd::uncompress is unavailable"); +} +Error zstd::uncompress(ArrayRef Input, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + llvm_unreachable("zstd::uncompress is unavailable"); +} +#endif diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in --- a/llvm/test/lit.site.cfg.py.in +++ b/llvm/test/lit.site.cfg.py.in @@ -37,6 +37,7 @@ config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@ config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.have_zlib = @LLVM_ENABLE_ZLIB@ +config.have_zstd = @LLVM_ENABLE_ZSTD@ config.have_libxar = @LLVM_HAVE_LIBXAR@ config.have_libxml2 = @LLVM_ENABLE_LIBXML2@ config.have_curl = @LLVM_ENABLE_CURL@ diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp --- a/llvm/unittests/Support/CompressionTest.cpp +++ b/llvm/unittests/Support/CompressionTest.cpp @@ -61,4 +61,42 @@ } #endif +#if LLVM_ENABLE_ZSTD +static void testZstdCompression(StringRef Input, int Level) { + SmallVector Compressed; + SmallVector Uncompressed; + zstd::compress(arrayRefFromStringRef(Input), Compressed, Level); + + // Check that uncompressed buffer is the same as original. + Error E = zstd::uncompress(Compressed, Uncompressed, Input.size()); + consumeError(std::move(E)); + + EXPECT_EQ(Input, toStringRef(Uncompressed)); + if (Input.size() > 0) { + // Uncompression fails if expected length is too short. + E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1); + EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E))); + } +} + +TEST(CompressionTest, Zstd) { + testZstdCompression("", zstd::DefaultCompression); + + testZstdCompression("hello, world!", zstd::NoCompression); + testZstdCompression("hello, world!", zstd::BestSizeCompression); + testZstdCompression("hello, world!", zstd::BestSpeedCompression); + testZstdCompression("hello, world!", zstd::DefaultCompression); + + const size_t kSize = 1024; + char BinaryData[kSize]; + for (size_t i = 0; i < kSize; ++i) + BinaryData[i] = i & 255; + StringRef BinaryDataStr(BinaryData, kSize); + + testZstdCompression(BinaryDataStr, zstd::NoCompression); + testZstdCompression(BinaryDataStr, zstd::BestSizeCompression); + testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression); + testZstdCompression(BinaryDataStr, zstd::DefaultCompression); +} +#endif } diff --git a/utils/bazel/llvm_configs/llvm-config.h.cmake b/utils/bazel/llvm_configs/llvm-config.h.cmake --- a/utils/bazel/llvm_configs/llvm-config.h.cmake +++ b/utils/bazel/llvm_configs/llvm-config.h.cmake @@ -95,6 +95,9 @@ /* Define if zlib compression is available */ #cmakedefine01 LLVM_ENABLE_ZLIB +/* Define if zstd compression is available */ +#cmakedefine01 LLVM_ENABLE_ZSTD + /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */ #cmakedefine LLVM_HAVE_TF_API