Build Windows releases with libxml enabled, to unbreak llvm-mt

[hans]

Summary

Recent CMake versions have started to prefer llvm-mt when using clang-cl, which doesn't work at all when llvm-mt is built without libxml which has been the case so far.

See https://github.com/llvm/llvm-project/issues/55817

[thakis]

Oh, cool!

I always imagined we'd (optionally, default-off, but then probably default-on on windows) ExternalProject_Add libxml from somewhere (similar to https://reviews.llvm.org/D122082#inline-1186216) – then this would be less bolted on. But doing this here first is better than what we currently have, so lg :)

[thakis]

(See also https://llvm.org/PR38966 for some past discussions on this.)

[thakis]

llvm/utils/release/build_llvm_release.bat

155

(why do we do this twice?)

155

(…oh, different bitness, nvm.)

[thakis]

In D129571#3645698, @thakis wrote:

Oh, cool!

I always imagined we'd (optionally, default-off, but then probably default-on on windows) ExternalProject_Add libxml from somewhere

(…or with https://cmake.org/cmake/help/latest/module/FetchContent.html)

[penzn]

This change LGTM, can we also add some description of this process to developer docs? Anyone who would building LLVM on Windows is going to be hit by llvm-mt errors in current Visual Studio versions.

[hans]

In D129571#3645827, @penzn wrote:

This change LGTM, can we also add some description of this process to developer docs? Anyone who would building LLVM on Windows is going to be hit by llvm-mt errors in current Visual Studio versions.

Yes, I'll try to follow up with that. It only hits users trying to use a self-built clang-cl to build something else that's configured with cmake though (such as when bootstrapping clang-cl), and I think for those cases a work-around such as -DCMAKE_MT=“C:/Program Files (x86)/Windows Kits/10/bin/10.0.19041.0/x64/mt.exe” is probably better than this giant kludge.

[CarlosAlbertoEnciso]

@hans Looking at your changes as part of https://reviews.llvm.org/D129559, the stage1 configuration fails with the error:

-- Looking for xmlReadMemory
-- Looking for xmlReadMemory - not found
CMake Error at cmake/config-ix.cmake:156 (message):
  Failed to configure libxml2

I think the issue is with the -DLIBXML2_INCLUDE_DIR. Looking at the CMake documentation:

LIBXML2_INCLUDE_DIR the directory containing LibXml2 headers
LIBXML2_INCLUDE_DIRS list of the include directories needed to use LibXml2

list(APPEND CMAKE_REQUIRED_INCLUDES ${LIBXML2_INCLUDE_DIRS})
list(APPEND CMAKE_REQUIRED_LIBRARIES ${LIBXML2_LIBRARIES})
...
check_symbol_exists(xmlReadMemory libxml/xmlreader.h HAVE_LIBXML2)
...
if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON AND NOT HAVE_LIBXML2)
  message(FATAL_ERROR "Failed to configure libxml2")
endif()

The CMake script is looking in ${LIBXML2_INCLUDE_DIRS}
Changing from -DLIBXML2_INCLUDE_DIR=.... to -DLIBXML2_INCLUDE_DIRS=... solved the issue and managed to build both 32/64 binaries.

[hans]

In D129571#3645827, @penzn wrote:

This change LGTM, can we also add some description of this process to developer docs? Anyone who would building LLVM on Windows is going to be hit by llvm-mt errors in current Visual Studio versions.

Sent https://reviews.llvm.org/D129770 for the docs update.

In D129571#3651781, @CarlosAlbertoEnciso wrote:

@hans Looking at your changes as part of https://reviews.llvm.org/D129559, the stage1 configuration fails with the error:

-- Looking for xmlReadMemory
-- Looking for xmlReadMemory - not found
CMake Error at cmake/config-ix.cmake:156 (message):
  Failed to configure libxml2

I think the issue is with the -DLIBXML2_INCLUDE_DIR. Looking at the CMake documentation:

LIBXML2_INCLUDE_DIR the directory containing LibXml2 headers
LIBXML2_INCLUDE_DIRS list of the include directories needed to use LibXml2

list(APPEND CMAKE_REQUIRED_INCLUDES ${LIBXML2_INCLUDE_DIRS})
list(APPEND CMAKE_REQUIRED_LIBRARIES ${LIBXML2_LIBRARIES})
...
check_symbol_exists(xmlReadMemory libxml/xmlreader.h HAVE_LIBXML2)
...
if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON AND NOT HAVE_LIBXML2)
  message(FATAL_ERROR "Failed to configure libxml2")
endif()

The CMake script is looking in ${LIBXML2_INCLUDE_DIRS}
Changing from -DLIBXML2_INCLUDE_DIR=.... to -DLIBXML2_INCLUDE_DIRS=... solved the issue and managed to build both 32/64 binaries.

It worked on my machine :-/ I suppose CMake may have changed (I used 3.20). Let me check if changing to LIBXML2_INCLUDE_DIRS works for me also...

[hans]

Let me check if changing to LIBXML2_INCLUDE_DIRS works for me also...

It does. Updated in 77350198d344f7c07ef221e967613c9e65f88aca

[CarlosAlbertoEnciso]

Thanks.

llvm/utils/release/build_llvm_release.bat

set cmake_flags=^
   -DLLVM_BUILD_LLVM_C_DYLIB=ON ^
   -DCMAKE_INSTALL_UCRT_LIBRARIES=ON ^
   -DPython3_FIND_REGISTRY=NEVER ^
   -DPACKAGE_VERSION=%package_version% ^
   -DLLDB_RELOCATABLE_PYTHON=1 ^
   -DLLDB_EMBED_PYTHON_HOME=OFF ^
   -DLLDB_TEST_COMPILER=%cd%\build32_stage0\bin\clang.exe ^
   -DCMAKE_CL_SHOWINCLUDES_PREFIX="Note: including file: " ^
+  -DLLVM_ENABLE_LIBXML2=FORCE_ON ^
+  -DLLDB_ENABLE_LIBXML2=OFF ^
+  -DCMAKE_C_FLAGS="-DLIBXML_STATIC" ^
+  -DCMAKE_CXX_FLAGS="-DLIBXML_STATIC" ^
   -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lld;compiler-rt;lldb;openmp"
 
 REM TODO: Run the "check-all" tests.
 
 set OLDPATH=%PATH%
 
+curl -O https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.12/libxml2-v2.9.12.tar.gz || exit /b
+tar zxf libxml2-v2.9.12.tar.gz
+
 set "VSCMD_START_DIR=%CD%"
 call "%vsdevcmd%" -arch=x86
 set PATH=%python32_dir%;%PATH%
 set CC=
 set CXX=
 mkdir build32_stage0
 cd build32_stage0
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% ..\llvm-project\llvm || exit /b
+
+mkdir libxmlbuild
+cd libxmlbuild
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_C14N=OFF -DLIBXML2_WITH_CATALOG=OFF -DLIBXML2_WITH_DEBUG=OFF -DLIBXML2_WITH_DOCB=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_HTML=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_ICONV=OFF -DLIBXML2_WITH_ICU=OFF -DLIBXML2_WITH_ISO8859X=OFF -DLIBXML2_WITH_LEGACY=OFF -DLIBXML2_WITH_LZMA=OFF -DLIBXML2_WITH_MEM_DEBUG=OFF -DLIBXML2_WITH_MODULES=OFF -DLIBXML2_WITH_OUTPUT=ON -DLIBXML2_WITH_PATTERN=OFF -DLIBXML2_WITH_PROGRAMS=OFF -DLIBXML2_WITH_PUSH=OFF -DLIBXML2_WITH_PYTHON=OFF -DLIBXML2_WITH_READER=OFF -DLIBXML2_WITH_REGEXPS=OFF -DLIBXML2_WITH_RUN_DEBUG=OFF -DLIBXML2_WITH_SAX1=OFF -DLIBXML2_WITH_SCHEMAS=OFF -DLIBXML2_WITH_SCHEMATRON=OFF -DLIBXML2_WITH_TESTS=OFF -DLIBXML2_WITH_THREADS=ON -DLIBXML2_WITH_THREAD_ALLOC=OFF -DLIBXML2_WITH_TREE=ON -DLIBXML2_WITH_VALID=OFF -DLIBXML2_WITH_WRITER=OFF -DLIBXML2_WITH_XINCLUDE=OFF -DLIBXML2_WITH_XPATH=OFF -DLIBXML2_WITH_XPTR=OFF -DLIBXML2_WITH_ZLIB=OFF ../../libxml2-v2.9.12 || exit /b
+ninja install || exit /b
+set libxmldir=%cd%\install
+set "libxmldir=%libxmldir:\=/%"
+cd ..
+
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
+
 ninja || ninja || ninja || exit /b
 REM ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 REM ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 REM ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 cd..
 
 mkdir build32
 cd build32
 set CC=..\build32_stage0\bin\clang-cl
 set CXX=..\build32_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPython3_ROOT_DIR=%python32_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 REM ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 REM ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 REM ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja package || exit /b
 cd ..
 
 set "VSCMD_START_DIR=%CD%"
 set PATH=%OLDPATH%
 call "%vsdevcmd%" -arch=amd64
 set PATH=%python64_dir%;%PATH%
 set CC=
 set CXX=
 mkdir build64_stage0
 cd build64_stage0
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% ..\llvm-project\llvm || exit /b
+
+mkdir libxmlbuild
+cd libxmlbuild
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_C14N=OFF -DLIBXML2_WITH_CATALOG=OFF -DLIBXML2_WITH_DEBUG=OFF -DLIBXML2_WITH_DOCB=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_HTML=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_ICONV=OFF -DLIBXML2_WITH_ICU=OFF -DLIBXML2_WITH_ISO8859X=OFF -DLIBXML2_WITH_LEGACY=OFF -DLIBXML2_WITH_LZMA=OFF -DLIBXML2_WITH_MEM_DEBUG=OFF -DLIBXML2_WITH_MODULES=OFF -DLIBXML2_WITH_OUTPUT=ON -DLIBXML2_WITH_PATTERN=OFF -DLIBXML2_WITH_PROGRAMS=OFF -DLIBXML2_WITH_PUSH=OFF -DLIBXML2_WITH_PYTHON=OFF -DLIBXML2_WITH_READER=OFF -DLIBXML2_WITH_REGEXPS=OFF -DLIBXML2_WITH_RUN_DEBUG=OFF -DLIBXML2_WITH_SAX1=OFF -DLIBXML2_WITH_SCHEMAS=OFF -DLIBXML2_WITH_SCHEMATRON=OFF -DLIBXML2_WITH_TESTS=OFF -DLIBXML2_WITH_THREADS=ON -DLIBXML2_WITH_THREAD_ALLOC=OFF -DLIBXML2_WITH_TREE=ON -DLIBXML2_WITH_VALID=OFF -DLIBXML2_WITH_WRITER=OFF -DLIBXML2_WITH_XINCLUDE=OFF -DLIBXML2_WITH_XPATH=OFF -DLIBXML2_WITH_XPTR=OFF -DLIBXML2_WITH_ZLIB=OFF ../../libxml2-v2.9.12 || exit /b
+ninja install || exit /b
+set libxmldir=%cd%\install
+set "libxmldir=%libxmldir:\=/%"
+cd ..
+
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja check-clangd || ninja check-clangd || ninja check-clangd || exit /b
 cd..
 
 mkdir build64
 cd build64
 set CC=..\build64_stage0\bin\clang-cl
 set CXX=..\build64_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% ..\llvm-project\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPython3_ROOT_DIR=%python64_dir% -DLIBXML2_INCLUDE_DIR=%libxmldir%/include/libxml2 -DLIBXML2_LIBRARIES=%libxmldir%/lib/libxml2s.lib ..\llvm-project\llvm || exit /b
 ninja || ninja || ninja || exit /b
 ninja check-llvm || ninja check-llvm || ninja check-llvm || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
 ninja check-lld || ninja check-lld || ninja check-lld || exit /b
 ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja check-clang-tools || ninja check-clang-tools || ninja check-clang-tools || exit /b
 ninja check-clangd || ninja check-clangd || ninja check-clangd || exit /b
 ninja package || exit /b
 cd ..