[CMake] Add ClangBootstrap configuration

[Amir]

Summary

Factor out the common parts of Clang bootstrap configuration into a separate
CMake module.

[Amir]

rebase

[phosek]

clang/CMakeLists.txt

912–926

I don't think we need a dedicated keyword for each tool, I'd just pass these through CMAKE_ARGS.

[Amir]

Move out toolchain tools back into CMAKE_ARGS

[phosek]

clang/cmake/modules/ClangBootstrap.cmake

11

We usually use lowercase names.

[Amir]

s/clang_Bootstrap_Add/clang_bootstrap_add

[Amir]

clang/cmake/modules/ClangBootstrap.cmake

11

Thanks! I was a bit confused by CMake's case use (e.g. ExternalProject_Add)

[thevinster]

clang/cmake/modules/ClangBootstrap.cmake

12

Were you planning to also use the single arguments list such as ARG_LINKER in the CMAKE_ARGS? Without it, I have to supply an override to CLANG_BOLT_INSTRUMENT_EXTRA_CMAKE_FLAGS so I can avoid using the gnu linker.

[Amir]

clang/cmake/modules/ClangBootstrap.cmake

12

Yes, I added those in the first version of the diff and just forgot to remove them. But as Peter mentioned:

I don't think we need a dedicated keyword for each tool, I'd just pass these through CMAKE_ARGS.

I'm neutral about adding ARG_LINKER or setting it through EXTRA_CMAKE_FLAGS, but I think explicit overrides for each tool are a bit too verbose.
Do you think having ARG_LINKER and passing the rest as EXTRA_CMAKE_FLAGS is a good tradeoff? cc @phosek

[Amir]

No longer needed for Clang-BOLT. @thevinster – feel free to commandeer if it fits your needs.

clang/CMakeLists.txt

foreach(variableName ${variableNames})
     endif()
     if(${variableName} AND variableName MATCHES "LLVM_EXTERNAL_.*_SOURCE_DIR")
       list(APPEND PASSTHROUGH_VARIABLES
         -D${variableName}=${${variableName}})
     endif()
   endforeach()
 
   # Build arguments for native tool used in CMake.
-  set(build_configuration "$<CONFIG>")
   set(build_tool_args "${LLVM_EXTERNAL_PROJECT_BUILD_TOOL_ARGS}")
   if(NOT build_tool_args STREQUAL "")
     string(PREPEND build_tool_args "-- ")
     separate_arguments(build_tool_args UNIX_COMMAND "${build_tool_args}")
   endif()
 
-  ExternalProject_Add(${NEXT_CLANG_STAGE}
+  include(ClangBootstrap)
+  clang_Bootstrap_Add(${NEXT_CLANG_STAGE}
     DEPENDS clang-bootstrap-deps
-    PREFIX ${NEXT_CLANG_STAGE}
-    SOURCE_DIR ${CMAKE_SOURCE_DIR}
-    STAMP_DIR ${STAMP_DIR}
-    BINARY_DIR ${BINARY_DIR}
-    EXCLUDE_FROM_ALL 1
+    TABLEGEN ${${CLANG_STAGE}_TABLEGEN}
+    LINKER   ${${CLANG_STAGE}_LINKER}
+    AR       ${${CLANG_STAGE}_AR}
+    RANLIB   ${${CLANG_STAGE}_RANLIB}
+    OBJCOPY  ${${CLANG_STAGE}_OBJCOPY}
+    STRIP    ${${CLANG_STAGE}_STRIP}
     CMAKE_ARGS
-                # We shouldn't need to set this here, but INSTALL_DIR doesn't
-                # seem to work, so instead I'm passing this through
-                -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
                 ${PASSTHROUGH_VARIABLES}
                 ${CLANG_BOOTSTRAP_CMAKE_ARGS}
                  -DCLANG_STAGE=${NEXT_CLANG_STAGE}
                 ${COMPILER_OPTIONS}
-                ${${CLANG_STAGE}_TABLEGEN}
                 ${LTO_LIBRARY} ${verbose} ${PGO_OPT}
-                ${${CLANG_STAGE}_LINKER}
-                ${${CLANG_STAGE}_AR}
-                ${${CLANG_STAGE}_RANLIB}
-                ${${CLANG_STAGE}_OBJCOPY}
-                ${${CLANG_STAGE}_STRIP}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR}
-                                   --config ${build_configuration}
-                                   ${build_tool_args}
-    INSTALL_COMMAND ""
-    STEP_TARGETS configure build
-    USES_TERMINAL_CONFIGURE 1
-    USES_TERMINAL_BUILD 1
-    USES_TERMINAL_INSTALL 1
-    LIST_SEPARATOR |
-    )
+    BUILD_TOOL_ARGS ${build_tool_args}
 
   # exclude really-install from main target
   set_target_properties(${NEXT_CLANG_STAGE} PROPERTIES _EP_really-install_EXCLUDE_FROM_MAIN On)
   ExternalProject_Add_Step(${NEXT_CLANG_STAGE} really-install
     COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install
     COMMENT "Performing install step for '${NEXT_CLANG_STAGE}'"
     DEPENDEES build
     USES_TERMINAL 1
add_custom_command(OUTPUT ${CLANGXX_INSTRUMENTED}
     COMMAND ${CMAKE_COMMAND} -E create_symlink
       ${CLANG_INSTRUMENTED}
       ${CLANGXX_INSTRUMENTED}
     COMMENT "Creating symlink from BOLT instrumented clang to clang++"
     VERBATIM
   )
 
   # Build specified targets with instrumented Clang to collect the profile
-  set(STAMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/bolt-instrumented-clang-stamps/)
-  set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/bolt-instrumented-clang-bins/)
-  set(build_configuration "$<CONFIG>")
-  include(ExternalProject)
-  ExternalProject_Add(bolt-instrumentation-profile
-    DEPENDS clang++-instrumented
-    PREFIX bolt-instrumentation-profile
-    SOURCE_DIR ${CMAKE_SOURCE_DIR}
-    STAMP_DIR ${STAMP_DIR}
-    BINARY_DIR ${BINARY_DIR}
-    EXCLUDE_FROM_ALL 1
-    CMAKE_ARGS
-                ${CLANG_BOLT_INSTRUMENT_EXTRA_CMAKE_FLAGS}
-                # We shouldn't need to set this here, but INSTALL_DIR doesn't
-                # seem to work, so instead I'm passing this through
-                -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
-                -DCMAKE_C_COMPILER=${CLANG_INSTRUMENTED}
-                -DCMAKE_CXX_COMPILER=${CLANGXX_INSTRUMENTED}
-                -DCMAKE_ASM_COMPILER=${CLANG_INSTRUMENTED}
-                -DCMAKE_ASM_COMPILER_ID=Clang
+  include(ClangBootstrap)
+  set(COMPILER_OPTIONS
+  )
+  clang_Bootstrap_Add(bolt-instrumentation-profile
+    DEPENDS clang++-instrumented
+    TABLEGEN
+            -DLLVM_TABLEGEN=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-tblgen
+            -DCLANG_TABLEGEN=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang-tblgen
+    LINKER  -DCMAKE_LINKER=${LLVM_RUNTIME_OUTPUT_INTDIR}/ld.lld
+    AR      -DCMAKE_AR=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ar
+    RANLIB  -DCMAKE_RANLIB=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-ranlib
+    OBJCOPY -DCMAKE_OBJCOPY=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-objcopy
+    STRIP   -DCMAKE_STRIP=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-strip
+    READELF -DCMAKE_READELF=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-readelf
+    CMAKE_ARGS
+                ${CLANG_BOLT_INSTRUMENT_EXTRA_CMAKE_FLAGS}
+                ${COMPILER_OPTIONS}
                 -DCMAKE_BUILD_TYPE=Release
                 -DLLVM_ENABLE_PROJECTS=${CLANG_BOLT_INSTRUMENT_PROJECTS}
                 -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGETS_TO_BUILD}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR}
-                                   --config ${build_configuration}
-                                   --target ${CLANG_BOLT_INSTRUMENT_TARGETS}
-    INSTALL_COMMAND ""
-    STEP_TARGETS configure build
-    USES_TERMINAL_CONFIGURE 1
-    USES_TERMINAL_BUILD 1
-    USES_TERMINAL_INSTALL 1
+    BUILD_TOOL_ARGS
   )
 
   # Merge profiles into one using merge-fdata
   add_custom_target(clang-bolt-profile
     DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/prof.fdata
   )
   add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/prof.fdata
     DEPENDS merge-fdata bolt-instrumentation-profile-build

clang/cmake/modules/ClangBootstrap.cmake

+include(ExternalProject)
+
+# clang_Bootstrap_Add(name ...
+#   DEPENDS targets...
+#     Targets that this project depends on
+#   TABLEGEN
+#   LINKER
+#   AR
+#   RANLIB
+#   OBJCOPY
+#   STRIP
+#     Toolchain binaries
+#   CMAKE_ARGS arguments...
+#     Optional cmake arguments to pass when configuring the project
+#   BUILD_TOOL_ARGS arguments...
+#     Optional arguments to pass to the build tool
+macro(clang_Bootstrap_Add name)
+  cmake_parse_arguments(ARG "" "LINKER;AR;RANLIB;OBJCOPY;STRIP"
+    "DEPENDS;TABLEGEN;CMAKE_ARGS;BUILD_TOOL_ARGS"
+    ${ARGN})
+  set(STAMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/${name}-stamps/)
+  set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${name}-bins/)
+  # Build arguments for native tool used in CMake.
+  set(build_configuration "$<CONFIG>")
+
+  ExternalProject_Add(${name}
+    DEPENDS ${ARG_DEPENDS}
+    PREFIX ${name}
+    SOURCE_DIR ${CMAKE_SOURCE_DIR}
+    STAMP_DIR ${STAMP_DIR}
+    BINARY_DIR ${BINARY_DIR}
+    EXCLUDE_FROM_ALL 1
+    CMAKE_ARGS
+                ${ARG_TABLEGEN}
+                ${ARG_LINKER}
+                ${ARG_AR}
+                ${ARG_RANLIB}
+                ${ARG_OBJCOPY}
+                ${ARG_STRIP}
+                # We shouldn't need to set this here, but INSTALL_DIR doesn't
+                # seem to work, so instead I'm passing this through
+                -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
+                ${ARG_CMAKE_ARGS}
+    BUILD_COMMAND ${CMAKE_COMMAND} --build ${BINARY_DIR}
+                                   --config ${build_configuration}
+                                   ${ARG_BUILD_TOOL_ARGS}
+    INSTALL_COMMAND ""
+    STEP_TARGETS configure build
+    USES_TERMINAL_CONFIGURE 1
+    USES_TERMINAL_BUILD 1
+    USES_TERMINAL_INSTALL 1
+    LIST_SEPARATOR |
+  )
+endmacro()