[CMake] Support cross-compiling with multi-stage builds

[phosek]

Summary

When using multi-stage builds, we would like support cross-compilation.
Example is 2-stage build when the first stage is compiled for host while
the second stage is compiled for the target.

Normally, the second stage would be also used for compiling runtimes,
but that's not possible when cross-compiling, so we use the first stage
compiler instead. However, we still want to use the second stage paths.
To do so, we set the -resource-dir of the first stage compiler to point
to the resource directory of the second stage.

We also need compiler tools that support the target architecture. These
tools are not guaranteed to be present on the host, but in case of
multi-stage build, we can build these tools in the first stage.

[beanz]

One slightly nitpicky comment.

llvm/cmake/modules/LLVMExternalProjectUtils.cmake

169

This could be problematic if the cross-compilation flow was ever hit using a non-clang compiler. I can't, off the top of my head, think of a use case where it would, but it might be worth checking CMAKE_<LANG>_COMPILER_ID to verify that it is *Clang.

[phosek]

Any other comments?

[beanz]

LGTM!

clang/CMakeLists.txt

if(APPLE)
       # variable, and have LLVM's CMake append the envar to the archiver calls.
       set(LTO_LIBRARY -DDARWIN_LTO_LIBRARY=${LLVM_SHLIB_OUTPUT_INTDIR}/libLTO.dylib
         -DDYLD_LIBRARY_PATH=${LLVM_LIBRARY_OUTPUT_INTDIR})
     elseif(NOT WIN32)
       add_dependencies(clang-bootstrap-deps llvm-ar llvm-ranlib)
       if(NOT BOOTSTRAP_LLVM_ENABLE_LLD AND LLVM_BINUTILS_INCDIR)
         add_dependencies(clang-bootstrap-deps LLVMgold)
       endif()
-      set(LTO_AR -DCMAKE_AR=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ar)
-      set(LTO_RANLIB -DCMAKE_RANLIB=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ranlib)
+      set(${CLANG_STAGE}_AR -DCMAKE_AR=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ar)
+      set(${CLANG_STAGE}_RANLIB -DCMAKE_RANLIB=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ranlib)
     endif()
   endif()
 
   if(CLANG_BOOTSTRAP_EXTRA_DEPS)
     add_dependencies(clang-bootstrap-deps ${CLANG_BOOTSTRAP_EXTRA_DEPS})
   endif()
 
   add_custom_target(${NEXT_CLANG_STAGE}-clear
if (CLANG_ENABLE_BOOTSTRAP)
   endif()
 
   set(COMPILER_OPTIONS
     -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/${CXX_COMPILER}
     -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/${C_COMPILER}
     -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/${C_COMPILER}
     -DCMAKE_ASM_COMPILER_ID=Clang)
 
+  if(BOOTSTRAP_CMAKE_SYSTEM_NAME)
+    set(${CLANG_STAGE}_CONFIG -DLLVM_CONFIG_PATH=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-config)
+    set(${CLANG_STAGE}_TABLEGEN
+      -DLLVM_TABLEGEN=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-tblgen
+      -DCLANG_TABLEGEN=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang-tblgen)
+    if(BOOTSTRAP_CMAKE_SYSTEM_NAME STREQUAL "Linux")
+      if(BOOTSTRAP_LLVM_ENABLE_LLD)
+        set(${CLANG_STAGE}_LINKER -DCMAKE_LINKER=${LLVM_RUNTIME_OUTPUT_INTDIR}/ld.lld)
+      endif()
+      if(NOT BOOTSTRAP_LLVM_ENABLE_LTO)
+        add_dependencies(clang-bootstrap-deps llvm-ar llvm-ranlib)
+        set(${CLANG_STAGE}_AR -DCMAKE_AR=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ar)
+        set(${CLANG_STAGE}_RANLIB -DCMAKE_RANLIB=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ranlib)
+      endif()
+      add_dependencies(clang-bootstrap-deps llvm-objcopy llvm-strip)
+      set(${CLANG_STAGE}_OBJCOPY -DCMAKE_OBJCOPY=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-objcopy)
+      set(${CLANG_STAGE}_STRIP -DCMAKE_STRIP=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-strip)
+    endif()
+  endif()
+
   if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps llvm-profdata)
     set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata)
   endif()
 
   if(LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps generate-profdata)
     set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata)
ExternalProject_Add(${NEXT_CLANG_STAGE}
     CMAKE_ARGS
                 # We shouldn't need to set this here, but INSTALL_DIR doesn't
                 # seem to work, so instead I'm passing this through
                 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
                 ${CLANG_BOOTSTRAP_CMAKE_ARGS}
                 ${PASSTHROUGH_VARIABLES}
                  -DCLANG_STAGE=${NEXT_CLANG_STAGE}
                 ${COMPILER_OPTIONS}
-                ${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT}
+                ${${CLANG_STAGE}_CONFIG}
+                ${${CLANG_STAGE}_TABLEGEN}
+                ${LTO_LIBRARY} ${verbose} ${PGO_OPT}
+                ${${CLANG_STAGE}_LINKER}
+                ${${CLANG_STAGE}_AR}
+                ${${CLANG_STAGE}_RANLIB}
+                ${${CLANG_STAGE}_OBJCOPY}
+                ${${CLANG_STAGE}_STRIP}
     INSTALL_COMMAND ""
     STEP_TARGETS configure build
     USES_TERMINAL_CONFIGURE 1
     USES_TERMINAL_BUILD 1
     USES_TERMINAL_INSTALL 1
     LIST_SEPARATOR |
     )
 

llvm/cmake/modules/LLVMExternalProjectUtils.cmake

set(compiler_args -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                       -DCMAKE_LINKER=${CMAKE_LINKER}
                       -DCMAKE_AR=${CMAKE_AR}
                       -DCMAKE_RANLIB=${CMAKE_RANLIB}
                       -DCMAKE_NM=${CMAKE_NM}
                       -DCMAKE_OBJCOPY=${CMAKE_OBJCOPY}
                       -DCMAKE_OBJDUMP=${CMAKE_OBJDUMP}
                       -DCMAKE_STRIP=${CMAKE_STRIP})
     set(llvm_config_path ${LLVM_CONFIG_PATH})
+
+    if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+      string(REGEX MATCH "[0-9]+\\.[0-9]+(\\.[0-9]+)?" CLANG_VERSION
+             ${PACKAGE_VERSION})
+      set(resource_dir "${LLVM_LIBRARY_DIR}/clang/${CLANG_VERSION}")
+      set(flag_types ASM C CXX MODULE_LINKER SHARED_LINKER EXE_LINKER)
+      foreach(type ${flag_types})
+        set(${type}_flag -DCMAKE_${type}_FLAGS=-resource-dir=${resource_dir})
+      endforeach()
+      string(REPLACE ";" "|" flag_string "${flag_types}")
+      foreach(arg ${ARG_CMAKE_ARGS})
+        if(arg MATCHES "^-DCMAKE_(${flag_string})_FLAGS")
+          foreach(type ${flag_types})
+            if(arg MATCHES "^-DCMAKE_${type}_FLAGS")
+              string(REGEX REPLACE "^-DCMAKE_${type}_FLAGS=(.*)$" "\\1" flag_value "${arg}")
+              set(${type}_flag "${${type}_flag} ${flag_value}")
+            endif()
+          endforeach()
+        else()
+          list(APPEND cmake_args ${arg})
+        endif()
+      endforeach()
+      foreach(type ${flag_types})
+        list(APPEND cmake_args ${${type}_flag})
+      endforeach()
+    endif()
   else()
     set(llvm_config_path "$<TARGET_FILE:llvm-config>")
+    set(cmake_args ${ARG_CMAKE_ARGS})
   endif()
 
   ExternalProject_Add(${name}
     DEPENDS ${ARG_DEPENDS} llvm-config
     ${name}-clobber
     PREFIX ${CMAKE_BINARY_DIR}/projects/${name}
     SOURCE_DIR ${source_dir}
     STAMP_DIR ${STAMP_DIR}
     BINARY_DIR ${BINARY_DIR}
     ${exclude}
     CMAKE_ARGS ${${nameCanon}_CMAKE_ARGS}
                ${compiler_args}
                -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
                ${sysroot_arg}
                -DLLVM_BINARY_DIR=${PROJECT_BINARY_DIR}
                -DLLVM_CONFIG_PATH=${llvm_config_path}
                -DLLVM_ENABLE_WERROR=${LLVM_ENABLE_WERROR}
                -DLLVM_HOST_TRIPLE=${LLVM_HOST_TRIPLE}
                -DLLVM_HAVE_LINK_VERSION_SCRIPT=${LLVM_HAVE_LINK_VERSION_SCRIPT}
                -DPACKAGE_VERSION=${PACKAGE_VERSION}
                -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
                -DCMAKE_EXPORT_COMPILE_COMMANDS=1
-               ${ARG_CMAKE_ARGS}
+               ${cmake_args}
                ${PASSTHROUGH_VARIABLES}
     INSTALL_COMMAND ""
     STEP_TARGETS configure build
     BUILD_ALWAYS 1
     USES_TERMINAL_CONFIGURE 1
     USES_TERMINAL_BUILD 1
     USES_TERMINAL_INSTALL 1
     LIST_SEPARATOR |