[Proof of concept] Add an opt-in way to statically link a hermetic libxml2

[thakis]

Summary

When LLVM_ENABLE_LIBXML2 is set to HERMETIC, we now download libxml2
at cmake time, build it at build time, and statically link to it
in llvm-mt.

This reverts https://reviews.llvm.org/D129571 and uses
LLVM_ENABLE_LIBXML2=HERMETIC instead for the Windows installer.

---

This works for llvm-mt, but c-index-test and lldb also pick up the hermetic libxml2, and since the built libxml2 is so minimal, fail to link.

But even if that worked, I'm not sure if this is worth it. It looks kind of janky: The external project build step runs on every build; every target depending on libxml needs some extra code to force the install step to run, cmake is a bit temperamental about this, and so on. Maybe this isn't all that much easier than what D129571 did.

Or maybe there's a better way for doing this that I can't see.

Maybe this should be WindowsManifest-specific and WindowsManifest should optionally depend on the hermetic stuff, and HermeticLibXML2.cmake shouldn't try to pretend its find_package()-compatible. Then lldb and c-index-test wouldn't pick up the hermetic lib.

I've spent more time on this than I should've, so uploading as-is. Maybe someone has an opinion on this.

[hans]

It certainly simplifies build_llvm_release.bat a lot :-)

nit: Calling this hermetic seems counter-intuitive here, since it's actually pulling in stuff from way outside the build -- the opposite of hermetic really. Maybe the flag value should be "DOWNLOAD" or something like that?

But yeah, also not sure whether this is worth it. I kind of wish llvm-mt didn't have this dependency and could use something simpler, but I don't know whether that's realistic.

llvm/CMakeLists.txt

 set(FFI_LIBRARY_DIR "" CACHE PATH "Additional directory, where CMake should search for libffi.so")
 set(FFI_INCLUDE_DIR "" CACHE PATH "Additional directory, where CMake should search for ffi.h or ffi/ffi.h")
 
 set(LLVM_TARGET_ARCH "host"
   CACHE STRING "Set target to use for LLVM JIT or use \"host\" for automatic detection.")
 
 option(LLVM_ENABLE_TERMINFO "Use terminfo database if available." ON)
 
-set(LLVM_ENABLE_LIBXML2 "ON" CACHE STRING "Use libxml2 if available. Can be ON, OFF, or FORCE_ON")
+set(LLVM_ENABLE_LIBXML2 "ON" CACHE STRING "Use libxml2 if available. Can be ON, OFF, FORCE_ON, or HERMETIC")
 
 option(LLVM_ENABLE_LIBEDIT "Use libedit if available." ON)
 
 option(LLVM_ENABLE_LIBPFM "Use libpfm for performance counters if available." ON)
 
 # On z/OS, threads cannot be used because TLS is not supported.
 if (CMAKE_SYSTEM_NAME MATCHES "OS390")
   option(LLVM_ENABLE_THREADS "Use threads if available." OFF)

llvm/cmake/config-ix.cmake

if(ZLIB_FOUND)
     if(LLVM_ENABLE_ZLIB STREQUAL FORCE_ON AND NOT HAVE_ZLIB)
       message(FATAL_ERROR "Failed to configure zlib")
     endif()
   endif()
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
 if(LLVM_ENABLE_LIBXML2)
-  if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
+  if(LLVM_ENABLE_LIBXML2 STREQUAL HERMETIC)
+    include(HermeticLibXML2)
+    set(LLVM_LIBXML2_IS_HERMETIC ON)
+  elseif(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
     find_package(LibXml2 REQUIRED)
   elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
     find_package(LibXml2)
   endif()
   if(LibXml2_FOUND)
     # Check if libxml2 we found is usable; for example, we may have found a 32-bit
     # library on a 64-bit system which would result in a link-time failure.
     cmake_push_check_state()

llvm/cmake/modules/HermeticLibXML2.cmake

+include(ExternalProject)
+
+# Once we require CMake 3.24+, we could use FetchContent_Declare() and
+# FetchContent_MakeAvailable() to intercept the normal find_package() call.
+
+set(HERMETIC_LIBXML2_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/libxml2-install")
+
+# Set variables that `find_package(LibXml2)` would set.
+set(LibXml2_FOUND ON)
+set(LIBXML2_INCLUDE_DIRS "${HERMETIC_LIBXML2_INSTALL_DIR}/include/libxml2")
+if (WIN32)
+  set(LIBXML2_LIBRARY "${HERMETIC_LIBXML2_INSTALL_DIR}/lib/libxml2s.lib")
+else ()
+  set(LIBXML2_LIBRARY "${HERMETIC_LIBXML2_INSTALL_DIR}/lib/libxml2.a")
+endif ()
+set(LIBXML2_LIBRARIES "${LIBXML2_LIBRARY}")
+set(LIBXML2_DEFINITIONS "LIBXML_STATIC")
+
+# Disable everything except WITH_TREE and WITH_OUTPUT, both needed by LLVM's
+# WindowsManifestMerger.
+# Also enable WITH_THREADS, else libxml doesn't compile on Linux.
+ExternalProject_Add(LibXML2
+  GIT_REPOSITORY    https://gitlab.gnome.org/GNOME/libxml2
+  GIT_TAG           v2.9.12
+  GIT_SHALLOW       ON
+  SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libxml2-src
+  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/libxml2-bin
+  INSTALL_DIR ${HERMETIC_LIBXML2_INSTALL_DIR}
+
+  # ExternalProject_Add() happens in a sub-build. Let the outer build know
+  # that the subbuild will create $LIBXML2_LIBRARY.
+  BUILD_BYPRODUCTS ${LIBXML2_LIBRARY}
+
+  CMAKE_ARGS
+    -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_INSTALL_PREFIX=${HERMETIC_LIBXML2_INSTALL_DIR}
+
+    -DLIBXML2_WITH_C14N=OFF
+    -DLIBXML2_WITH_CATALOG=OFF
+    -DLIBXML2_WITH_DEBUG=OFF
+    -DLIBXML2_WITH_DOCB=OFF
+    -DLIBXML2_WITH_FTP=OFF
+    -DLIBXML2_WITH_HTML=OFF
+    -DLIBXML2_WITH_HTTP=OFF
+    -DLIBXML2_WITH_ICONV=OFF
+    -DLIBXML2_WITH_ICU=OFF
+    -DLIBXML2_WITH_ISO8859X=OFF
+    -DLIBXML2_WITH_LEGACY=OFF
+    -DLIBXML2_WITH_LZMA=OFF
+    -DLIBXML2_WITH_MEM_DEBUG=OFF
+    -DLIBXML2_WITH_MODULES=OFF
+    -DLIBXML2_WITH_OUTPUT=ON
+    -DLIBXML2_WITH_PATTERN=OFF
+    -DLIBXML2_WITH_PROGRAMS=OFF
+    -DLIBXML2_WITH_PUSH=OFF
+    -DLIBXML2_WITH_PYTHON=OFF
+    -DLIBXML2_WITH_READER=OFF
+    -DLIBXML2_WITH_REGEXPS=OFF
+    -DLIBXML2_WITH_RUN_DEBUG=OFF
+    -DLIBXML2_WITH_SAX1=OFF
+    -DLIBXML2_WITH_SCHEMAS=OFF
+    -DLIBXML2_WITH_SCHEMATRON=OFF
+    -DLIBXML2_WITH_TESTS=OFF
+    -DLIBXML2_WITH_THREADS=ON
+    -DLIBXML2_WITH_THREAD_ALLOC=OFF
+    -DLIBXML2_WITH_TREE=ON
+    -DLIBXML2_WITH_VALID=OFF
+    -DLIBXML2_WITH_WRITER=OFF
+    -DLIBXML2_WITH_XINCLUDE=OFF
+    -DLIBXML2_WITH_XPATH=OFF
+    -DLIBXML2_WITH_XPTR=OFF
+    -DLIBXML2_WITH_ZLIB=OFF
+  )
+

llvm/lib/WindowsManifest/CMakeLists.txt

 include(GetLibraryName)
 
 if(LLVM_ENABLE_LIBXML2)
-  set(imported_libs LibXml2::LibXml2)
+  set(imported_libs ${LIBXML2_LIBRARIES})
+  include_directories(BEFORE ${LIBXML2_INCLUDE_DIRS})
 endif()
 
 add_llvm_component_library(LLVMWindowsManifest
   WindowsManifestMerger.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/WindowsManifest
   ${Backtrace_INCLUDE_DIRS}
 
   LINK_LIBS
   ${imported_libs}
 
   LINK_COMPONENTS
   Support
   )
 
+if (LLVM_ENABLE_LIBXML2 AND LLVM_LIBXML2_IS_HERMETIC)
+  # Force hermetic LibXML2's install step to run.
+  add_dependencies(LLVMWindowsManifest LibXML2)
+endif()
+
 # This block is only needed for llvm-config. When we deprecate llvm-config and
 # move to using CMake export, this block can be removed.
 if(LLVM_ENABLE_LIBXML2)
-  # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators.
-  if(CMAKE_BUILD_TYPE)
-    string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
-    get_property(libxml2_library TARGET LibXml2::LibXml2 PROPERTY LOCATION_${build_type})
-  endif()
-  if(NOT libxml2_library)
-    get_property(libxml2_library TARGET LibXml2::LibXml2 PROPERTY LOCATION)
-  endif()
-  get_library_name(${libxml2_library} libxml2_library)
-  set_property(TARGET LLVMWindowsManifest PROPERTY LLVM_SYSTEM_LIBS ${libxml2_library})
+  get_library_name(${LIBXML2_LIBRARY} libxml2_library_name)
+  set_property(TARGET LLVMWindowsManifest PROPERTY LLVM_SYSTEM_LIBS ${libxml2_library_name})
 endif()

llvm/utils/release/build_llvm_release.bat

 mv llvm-project-* llvm-project || exit /b
 
 REM Setting CMAKE_CL_SHOWINCLUDES_PREFIX to work around PR27226.
 set cmake_flags=^
   -DCMAKE_BUILD_TYPE=Release ^
   -DLLVM_ENABLE_ASSERTIONS=OFF ^
   -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON ^
   -DLLVM_BUILD_LLVM_C_DYLIB=ON ^
+  -DLLVM_ENABLE_LIBXML2=HERMETIC ^
   -DCMAKE_INSTALL_UCRT_LIBRARIES=ON ^
   -DPython3_FIND_REGISTRY=NEVER ^
   -DPACKAGE_VERSION=%package_version% ^
   -DLLDB_RELOCATABLE_PYTHON=1 ^
   -DLLDB_EMBED_PYTHON_HOME=OFF ^
   -DLLDB_TEST_COMPILER=%cd%\build32_stage0\bin\clang.exe ^
   -DCMAKE_CL_SHOWINCLUDES_PREFIX="Note: including file: " ^
-  -DLLVM_ENABLE_LIBXML2=FORCE_ON ^
-  -DLLDB_ENABLE_LIBXML2=OFF ^
-  -DCMAKE_C_FLAGS="-DLIBXML_STATIC" ^
-  -DCMAKE_CXX_FLAGS="-DLIBXML_STATIC" ^
   -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lld;compiler-rt;lldb;openmp"
 
 REM TODO: Run the "check-all" tests.
 
 set OLDPATH=%PATH%
 
-curl -O https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.12/libxml2-v2.9.12.tar.gz || exit /b
-tar zxf libxml2-v2.9.12.tar.gz
-
 set "VSCMD_START_DIR=%CD%"
 call "%vsdevcmd%" -arch=x86
 set PATH=%python32_dir%;%PATH%
 set CC=
 set CXX=
 mkdir build32_stage0
 cd build32_stage0
-
-mkdir libxmlbuild
-cd libxmlbuild
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_C14N=OFF -DLIBXML2_WITH_CATALOG=OFF -DLIBXML2_WITH_DEBUG=OFF -DLIBXML2_WITH_DOCB=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_HTML=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_ICONV=OFF -DLIBXML2_WITH_ICU=OFF -DLIBXML2_WITH_ISO8859X=OFF -DLIBXML2_WITH_LEGACY=OFF -DLIBXML2_WITH_LZMA=OFF -DLIBXML2_WITH_MEM_DEBUG=OFF -DLIBXML2_WITH_MODULES=OFF -DLIBXML2_WITH_OUTPUT=ON -DLIBXML2_WITH_PATTERN=OFF -DLIBXML2_WITH_PROGRAMS=OFF -DLIBXML2_WITH_PUSH=OFF -DLIBXML2_WITH_PYTHON=OFF -DLIBXML2_WITH_READER=OFF -DLIBXML2_WITH_REGEXPS=OFF -DLIBXML2_WITH_RUN_DEBUG=OFF -DLIBXML2_WITH_SAX1=OFF -DLIBXML2_WITH_SCHEMAS=OFF -DLIBXML2_WITH_SCHEMATRON=OFF -DLIBXML2_WITH_TESTS=OFF -DLIBXML2_WITH_THREADS=ON -DLIBXML2_WITH_THREAD_ALLOC=OFF -DLIBXML2_WITH_TREE=ON -DLIBXML2_WITH_VALID=OFF -DLIBXML2_WITH_WRITER=OFF -DLIBXML2_WITH_XINCLUDE=OFF -DLIBXML2_WITH_XPATH=OFF -DLIBXML2_WITH_XPTR=OFF -DLIBXML2_WITH_ZLIB=OFF ../../libxml2-v2.9.12 || exit /b
-ninja install || exit /b
-set libxmldir=%cd%\install
-set "libxmldir=%libxmldir:\=/%"
-cd ..
-
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% ..\llvm-project\llvm || exit /b
 
 ninja || ninja || ninja || exit /b
 REM ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 REM ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 REM ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 cd..
 
 mkdir build32
 cd build32
 set CC=..\build32_stage0\bin\clang-cl
 set CXX=..\build32_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 REM ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 REM ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 REM ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja package || exit /b
 cd ..
 
 set "VSCMD_START_DIR=%CD%"
 set PATH=%OLDPATH%
 call "%vsdevcmd%" -arch=amd64
 set PATH=%python64_dir%;%PATH%
 set CC=
 set CXX=
 mkdir build64_stage0
 cd build64_stage0
-
-mkdir libxmlbuild
-cd libxmlbuild
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_C14N=OFF -DLIBXML2_WITH_CATALOG=OFF -DLIBXML2_WITH_DEBUG=OFF -DLIBXML2_WITH_DOCB=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_HTML=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_ICONV=OFF -DLIBXML2_WITH_ICU=OFF -DLIBXML2_WITH_ISO8859X=OFF -DLIBXML2_WITH_LEGACY=OFF -DLIBXML2_WITH_LZMA=OFF -DLIBXML2_WITH_MEM_DEBUG=OFF -DLIBXML2_WITH_MODULES=OFF -DLIBXML2_WITH_OUTPUT=ON -DLIBXML2_WITH_PATTERN=OFF -DLIBXML2_WITH_PROGRAMS=OFF -DLIBXML2_WITH_PUSH=OFF -DLIBXML2_WITH_PYTHON=OFF -DLIBXML2_WITH_READER=OFF -DLIBXML2_WITH_REGEXPS=OFF -DLIBXML2_WITH_RUN_DEBUG=OFF -DLIBXML2_WITH_SAX1=OFF -DLIBXML2_WITH_SCHEMAS=OFF -DLIBXML2_WITH_SCHEMATRON=OFF -DLIBXML2_WITH_TESTS=OFF -DLIBXML2_WITH_THREADS=ON -DLIBXML2_WITH_THREAD_ALLOC=OFF -DLIBXML2_WITH_TREE=ON -DLIBXML2_WITH_VALID=OFF -DLIBXML2_WITH_WRITER=OFF -DLIBXML2_WITH_XINCLUDE=OFF -DLIBXML2_WITH_XPATH=OFF -DLIBXML2_WITH_XPTR=OFF -DLIBXML2_WITH_ZLIB=OFF ../../libxml2-v2.9.12 || exit /b
-ninja install || exit /b
-set libxmldir=%cd%\install
-set "libxmldir=%libxmldir:\=/%"
-cd ..
-
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja check-clangd || ninja check-clangd || ninja check-clangd || exit /b
 cd..
 
 mkdir build64
 cd build64
 set CC=..\build64_stage0\bin\clang-cl
 set CXX=..\build64_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja check-clangd || ninja check-clangd || ninja check-clangd || exit /b
 ninja package || exit /b
 cd ..