[Clang][CMake] Allow using passthrough BOLT in BOLT-PGO.cmake

[Amir]

Summary

Factor out bolt project dependency from Clang-BOLT usage.
Allow using

• user-specified BOLT: LLVM_BOLT and MERGE_FDATA variables,
• BOLT from system path (with CMake's find_program),
• built from bolt project (the only previously supported way).

Customize BOLT-PGO.cmake cache file to build BOLT in first-stage build, and
passthrough and use it in the final clang-bolt invocation (on top of PGO binary).
This avoids building PGO'd BOLT (first instrumented and then optimized), cutting
the build time.

[Amir]

Add clang-bolt-profile-deps dependency

clang/CMakeLists.txt

if(BOOTSTRAP_CMAKE_SYSTEM_NAME STREQUAL "Linux")
       endif()
       add_dependencies(clang-bootstrap-deps llvm-objcopy llvm-strip)
       set(${CLANG_STAGE}_OBJCOPY -DCMAKE_OBJCOPY=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-objcopy)
       set(${CLANG_STAGE}_STRIP -DCMAKE_STRIP=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-strip)
       set(${CLANG_STAGE}_READELF -DCMAKE_READELF=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-readelf)
     endif()
   endif()
 
+  if(BOOTSTRAP_LLVM_BOLT)
+    add_dependencies(clang-bootstrap-deps llvm-bolt merge-fdata)
+    set(LLVM_BOLT ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-bolt)
+    set(MERGE_FDATA ${LLVM_RUNTIME_OUTPUT_INTDIR}/merge-fdata)
+    list(APPEND _BOOTSTRAP_DEFAULT_PASSTHROUGH
+      LLVM_BOLT
+      MERGE_FDATA
+    )
+  endif()
+
   if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps llvm-profdata)
     set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata)
   endif()
 
   if(LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps generate-profdata)
     set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata)
if (CLANG_ENABLE_BOOTSTRAP)
   endforeach()
 endif()
 
 if (CLANG_BOLT_INSTRUMENT AND NOT LLVM_BUILD_INSTRUMENTED)
   set(CLANG_PATH ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)
   set(CLANG_INSTRUMENTED ${CLANG_PATH}-bolt.inst)
   set(BOLT_FDATA ${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/prof.fdata)
 
+  add_custom_target(bolt-clang-instrumented-deps)
+  if ("bolt" IN_LIST LLVM_ENABLE_PROJECTS)
+    add_dependencies(bolt-clang-instrumented-deps llvm-bolt)
+    set(LLVM_BOLT $<TARGET_FILE:llvm-bolt>)
+  elseif (NOT LLVM_BOLT)
+    find_program(LLVM_BOLT llvm-bolt)
+  endif()
+
+  if (NOT LLVM_BOLT AND NOT "bolt" IN_LIST LLVM_ENABLE_PROJECTS)
+    message(STATUS "To enable optimizing Clang with BOLT enable bolt project or set LLVM_BOLT")
+  else()
     # Instrument clang with BOLT
     add_custom_target(clang-instrumented
       DEPENDS ${CLANG_INSTRUMENTED}
     )
     add_custom_command(OUTPUT ${CLANG_INSTRUMENTED}
-    DEPENDS clang llvm-bolt
-    COMMAND llvm-bolt ${CLANG_PATH} -o ${CLANG_INSTRUMENTED}
+      DEPENDS bolt-clang-instrumented-deps clang
+      COMMAND ${LLVM_BOLT} ${CLANG_PATH} -o ${CLANG_INSTRUMENTED}
         -instrument --instrumentation-file-append-pid
         --instrumentation-file=${BOLT_FDATA}
       COMMENT "Instrumenting clang binary with BOLT"
       VERBATIM
     )
 
     # Optimize original (pre-bolt) Clang using the collected profile
     set(CLANG_OPTIMIZED ${CMAKE_CURRENT_BINARY_DIR}/clang.bolt)
     add_custom_target(clang-bolt
       DEPENDS ${CLANG_OPTIMIZED}
     )
     add_custom_command(OUTPUT ${CLANG_OPTIMIZED}
       DEPENDS clang-bolt-profile
-    COMMAND llvm-bolt ${CLANG_PATH}
+      COMMAND ${LLVM_BOLT} ${CLANG_PATH}
         -o ${CLANG_OPTIMIZED}
         -data ${BOLT_FDATA}
         -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions
         -split-all-cold -split-eh -dyno-stats -icf=1 -use-gnu-stack
       COMMAND ${CMAKE_COMMAND} -E rename ${CLANG_OPTIMIZED} $<TARGET_FILE:clang>
       COMMENT "Optimizing Clang with BOLT"
       VERBATIM
     )
   endif()
+endif()
 
 if (LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION)
   add_subdirectory(utils/ClangVisualizers)
 endif()
 add_subdirectory(utils/hmaptool)
 
 if(CLANG_BUILT_STANDALONE)
   llvm_distribution_add_targets()
   process_llvm_pass_plugins()
 endif()
 
 set(CLANG_INSTALL_LIBDIR_BASENAME "lib${CLANG_LIBDIR_SUFFIX}")
 
 configure_file(
   ${CLANG_SOURCE_DIR}/include/clang/Config/config.h.cmake
   ${CLANG_BINARY_DIR}/include/clang/Config/config.h)

clang/cmake/caches/BOLT-PGO-stage2.cmake

+set(CMAKE_BUILD_TYPE Release CACHE STRING "")
+set(CLANG_BOLT_INSTRUMENT ON CACHE BOOL "")
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--emit-relocs,-znow" CACHE STRING "")
+
+set(LLVM_ENABLE_PROJECTS "clang" CACHE STRING "")
+set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
+
+# setup toolchain
+set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")
+set(LLVM_DISTRIBUTION_COMPONENTS
+  clang
+  clang-resource-headers
+  CACHE STRING "")

clang/cmake/caches/BOLT-PGO.cmake

 set(LLVM_ENABLE_PROJECTS "bolt;clang;lld" CACHE STRING "")
 
 set(CLANG_BOOTSTRAP_TARGETS
   stage2-clang-bolt
   stage2-distribution
   stage2-install-distribution
   CACHE STRING "")
 set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
   clang-bolt
   distribution
   install-distribution
   CACHE STRING "")
 
+set(BOOTSTRAP_LLVM_BOLT ON)
+
 set(PGO_BUILD_CONFIGURATION
-  ${CMAKE_CURRENT_LIST_DIR}/BOLT.cmake
+  ${CMAKE_CURRENT_LIST_DIR}/BOLT-PGO-stage2.cmake
   CACHE STRING "")
 include(${CMAKE_CURRENT_LIST_DIR}/PGO.cmake)

clang/utils/perf-training/CMakeLists.txt

add_lit_testsuite(generate-bolt-fdata "Generating BOLT profile for Clang"
     EXCLUDE_FROM_CHECK_ALL
     DEPENDS clang-instrumented clear-bolt-fdata
     )
 
   add_custom_target(clear-bolt-fdata
     COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} fdata
     COMMENT "Clearing old BOLT fdata")
 
+  add_custom_target(clang-bolt-profile-deps)
+  if ("bolt" IN_LIST LLVM_ENABLE_PROJECTS)
+    add_dependencies(clang-bolt-profile-deps merge-fdata)
+    set(MERGE_FDATA $<TARGET_FILE:merge-fdata>)
+  elseif (NOT MERGE_FDATA)
+    find_program(MERGE_FDATA merge-fdata)
+  endif()
+
+  if (NOT MERGE_FDATA AND NOT "bolt" IN_LIST LLVM_ENABLE_PROJECTS)
+    message(STATUS "To enable optimizing Clang with BOLT enable bolt project or set MERGE_FDATA")
+  else()
     # Merge profiles into one using merge-fdata
     add_custom_target(clang-bolt-profile
-    COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge-fdata $<TARGET_FILE:merge-fdata> ${CMAKE_CURRENT_BINARY_DIR}/prof.fdata ${CMAKE_CURRENT_BINARY_DIR}
+      COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge-fdata ${MERGE_FDATA} ${CMAKE_CURRENT_BINARY_DIR}/prof.fdata ${CMAKE_CURRENT_BINARY_DIR}
       COMMENT "Merging BOLT fdata"
-    DEPENDS merge-fdata generate-bolt-fdata)
+      DEPENDS clang-bolt-profile-deps generate-bolt-fdata)
+  endif()
 endif()