Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -29,6 +29,7 @@ CLANGD_MALLOC_TRIM CLANGD_TIDY_CHECKS LLVM_ENABLE_ZLIB + LLVM_ENABLE_ZSTD ) configure_file( Index: clang-tools-extra/clangd/test/lit.site.cfg.py.in =================================================================== --- clang-tools-extra/clangd/test/lit.site.cfg.py.in +++ clang-tools-extra/clangd/test/lit.site.cfg.py.in @@ -17,6 +17,7 @@ config.clangd_enable_remote = @CLANGD_ENABLE_REMOTE@ config.clangd_tidy_checks = @CLANGD_TIDY_CHECKS@ config.have_zlib = @LLVM_ENABLE_ZLIB@ +config.have_zstd = @LLVM_ENABLE_ZSTD@ # Delegate logic to lit.cfg.py. lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py") Index: clang/test/CMakeLists.txt =================================================================== --- clang/test/CMakeLists.txt +++ clang/test/CMakeLists.txt @@ -11,6 +11,7 @@ CLANG_SPAWN_CC1 ENABLE_BACKTRACES LLVM_ENABLE_ZLIB + LLVM_ENABLE_ZSTD LLVM_ENABLE_PER_TARGET_RUNTIME_DIR LLVM_ENABLE_THREADS LLVM_WITH_Z3 Index: clang/test/lit.site.cfg.py.in =================================================================== --- clang/test/lit.site.cfg.py.in +++ clang/test/lit.site.cfg.py.in @@ -21,6 +21,7 @@ config.host_cxx = "@CMAKE_CXX_COMPILER@" config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" config.have_zlib = @LLVM_ENABLE_ZLIB@ +config.have_zstd = @LLVM_ENABLE_ZSTD@ config.clang_arcmt = @CLANG_ENABLE_ARCMT@ config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@ config.clang_enable_opaque_pointers = @CLANG_ENABLE_OPAQUE_POINTERS_INTERNAL@ Index: compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh =================================================================== --- compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -138,6 +138,7 @@ -DCMAKE_CXX_FLAGS_RELEASE="${LLVM_FLAGS}" \ -DLLVM_TABLEGEN=$TBLGEN \ -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_ENABLE_ZSTD=ON \ -DLLVM_ENABLE_TERMINFO=OFF \ -DLLVM_ENABLE_THREADS=OFF \ $LLVM_SRC Index: compiler-rt/test/lit.common.configured.in =================================================================== --- compiler-rt/test/lit.common.configured.in +++ compiler-rt/test/lit.common.configured.in @@ -65,6 +65,7 @@ set_default("target_suffix", "-%s" % config.target_arch) set_default("have_zlib", "@LLVM_ENABLE_ZLIB@") +set_default("have_zstd", "@LLVM_ENABLE_ZSTD@") set_default("libcxx_used", "@LLVM_LIBCXX_USED@") # LLVM tools dir can be passed in lit parameters, so try to Index: flang/CMakeLists.txt =================================================================== --- flang/CMakeLists.txt +++ flang/CMakeLists.txt @@ -70,6 +70,12 @@ if(LLVM_ENABLE_ZLIB) find_package(ZLIB REQUIRED) endif() + + # If LLVM links to zstd we need the imported targets so we can too. + if(LLVM_ENABLE_ZSTD) + find_package(ZSTD REQUIRED) + endif() + option(LLVM_ENABLE_PEDANTIC "Compile with pedantic enabled." ON) if(CMAKE_COMPILER_IS_GNUCXX) set(USE_NO_MAYBE_UNINITIALIZED 1) Index: lld/ELF/CMakeLists.txt =================================================================== --- lld/ELF/CMakeLists.txt +++ lld/ELF/CMakeLists.txt @@ -6,6 +6,10 @@ list(APPEND imported_libs ZLIB::ZLIB) endif() +if(LLVM_ENABLE_ZSTD) + list(APPEND imported_libs zstd) +endif() + add_lld_library(lldELF AArch64ErrataFix.cpp Arch/AArch64.cpp Index: lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt =================================================================== --- lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt +++ lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt @@ -18,6 +18,10 @@ list(APPEND LLDB_SYSTEM_LIBS ZLIB::ZLIB) endif() +if(LLVM_ENABLE_ZSTD) + list(APPEND LLDB_SYSTEM_LIBS zstd) +endif() + add_lldb_library(lldbPluginProcessGDBRemote PLUGIN GDBRemoteClientBase.cpp GDBRemoteCommunication.cpp Index: llvm/CMakeLists.txt =================================================================== --- llvm/CMakeLists.txt +++ llvm/CMakeLists.txt @@ -438,6 +438,8 @@ set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON") +set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON") + set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON") set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.") Index: llvm/cmake/config-ix.cmake =================================================================== --- llvm/cmake/config-ix.cmake +++ llvm/cmake/config-ix.cmake @@ -135,6 +135,27 @@ set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}") endif() +if(LLVM_ENABLE_ZSTD) + if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON) + find_package(ZSTD REQUIRED) + elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*") + find_package(ZSTD) + endif() + if(ZSTD_FOUND) + # Check if zstd we found is usable; for example, we may have found a 32-bit + # library on a 64-bit system which would result in a link-time failure. + cmake_push_check_state() + list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR}) + list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY}) + check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD) + cmake_pop_check_state() + if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD) + message(FATAL_ERROR "Failed to configure zstd") + endif() + endif() + set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}") +endif() + if(LLVM_ENABLE_LIBXML2) if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON) find_package(LibXml2 REQUIRED) Index: llvm/cmake/modules/FindZSTD.cmake =================================================================== --- /dev/null +++ llvm/cmake/modules/FindZSTD.cmake @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# - Try to find Facebook zstd library +# This will define +# ZSTD_FOUND +# ZSTD_INCLUDE_DIR +# ZSTD_LIBRARY +# + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) + +include(SelectLibraryConfigurations) +SELECT_LIBRARY_CONFIGURATIONS(ZSTD) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( + ZSTD DEFAULT_MSG + ZSTD_LIBRARY ZSTD_INCLUDE_DIR +) + +if (ZSTD_FOUND) + message(STATUS "Found Zstd: ${ZSTD_LIBRARY}") +endif() + +mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) Index: llvm/cmake/modules/LLVMConfig.cmake.in =================================================================== --- llvm/cmake/modules/LLVMConfig.cmake.in +++ llvm/cmake/modules/LLVMConfig.cmake.in @@ -73,6 +73,12 @@ find_package(ZLIB) endif() +set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@) +if(LLVM_ENABLE_ZSTD) + set(ZSTD_ROOT @ZSTD_ROOT@) + find_package(ZSTD) +endif() + set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@) if(LLVM_ENABLE_LIBXML2) find_package(LibXml2) Index: llvm/include/llvm/Config/llvm-config.h.cmake =================================================================== --- llvm/include/llvm/Config/llvm-config.h.cmake +++ llvm/include/llvm/Config/llvm-config.h.cmake @@ -92,6 +92,9 @@ /* Define if zlib compression is available */ #cmakedefine01 LLVM_ENABLE_ZLIB +/* Define if zstd compression is available */ +#cmakedefine01 LLVM_ENABLE_ZSTD + /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */ #cmakedefine LLVM_HAVE_TF_API Index: llvm/include/llvm/Support/Compression.h =================================================================== --- llvm/include/llvm/Support/Compression.h +++ llvm/include/llvm/Support/Compression.h @@ -44,7 +44,33 @@ } // End of namespace zlib +namespace zstd { + +static constexpr std::string AlgorithmName = "zstd"; +static constexpr int NoCompression = -5; +static constexpr int BestSpeedCompression = 1; +static constexpr int DefaultCompression = 5; +static constexpr int BestSizeCompression = 12; + +bool isAvailable(); + +void compress(StringRef InputBuffer, SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression); + +Error uncompress(StringRef InputBuffer, char *UncompressedBuffer, + size_t &UncompressedSize); + +Error uncompress(StringRef InputBuffer, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize); + +} // End of namespace zstd + +#if LLVM_ENABLE_ZSTD +namespace tooling = llvm::compression::zstd; +#else namespace tooling = llvm::compression::zlib; +#endif namespace profile = llvm::compression::tooling; Index: llvm/lib/Support/Compression.cpp =================================================================== --- llvm/lib/Support/Compression.cpp +++ llvm/lib/Support/Compression.cpp @@ -20,6 +20,9 @@ #if LLVM_ENABLE_ZLIB #include #endif +#if LLVM_ENABLE_ZSTD +#include +#endif using namespace llvm; @@ -102,3 +105,63 @@ llvm_unreachable("zlib::uncompress is unavailable"); } #endif + +#if LLVM_ENABLE_ZSTD + +bool zstd::isAvailable() { return true; } + +void zstd::compress(StringRef InputBuffer, + SmallVectorImpl &CompressedBuffer, int Level) { + unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size()); + CompressedBuffer.resize_for_overwrite(CompressedBufferSize); + unsigned long CompressedSize = ::ZSTD_compress( + (Bytef *)CompressedBuffer.data(), CompressedBufferSize, + (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level); + if (ZSTD_isError(CompressedSize)) + report_bad_alloc_error("Allocation failed"); + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + CompressedBuffer.truncate(CompressedSize); +} + +Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer, + size_t &UncompressedSize) { + unsigned long long const rSize = ZSTD_getFrameContentSize( + (const Bytef *)InputBuffer.data(), InputBuffer.size()); + size_t const Res = + ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize, + (const Bytef *)InputBuffer.data(), InputBuffer.size()); + UncompressedSize = Res; + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(UncompressedBuffer, UncompressedSize); + return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success(); +} + +Error zstd::uncompress(StringRef InputBuffer, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + UncompressedBuffer.resize_for_overwrite(UncompressedSize); + Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(), + UncompressedSize); + UncompressedBuffer.truncate(UncompressedSize); + return E; +} + +#else +bool zstd::isAvailable() { return false; } +void zstd::compress(StringRef InputBuffer, + SmallVectorImpl &CompressedBuffer, int Level) { + llvm_unreachable("zstd::compress is unavailable"); +} +Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer, + size_t &UncompressedSize) { + llvm_unreachable("zstd::uncompress is unavailable"); +} +Error zstd::uncompress(StringRef InputBuffer, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + llvm_unreachable("zstd::uncompress is unavailable"); +} +#endif \ No newline at end of file Index: utils/bazel/llvm_configs/llvm-config.h.cmake =================================================================== --- utils/bazel/llvm_configs/llvm-config.h.cmake +++ utils/bazel/llvm_configs/llvm-config.h.cmake @@ -92,6 +92,9 @@ /* Define if zlib compression is available */ #cmakedefine01 LLVM_ENABLE_ZLIB +/* Define if zstd compression is available */ +#cmakedefine01 LLVM_ENABLE_ZSTD + /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */ #cmakedefine LLVM_HAVE_TF_API