[Clang][CMake] Add CSSPGO support to LLVM_BUILD_INSTRUMENTED

[Amir]

Summary

Build on Clang-BOLT infrastructure to collect sample profiles for CSSPGO.
Add clang/cmake/caches/CSSPGO.cmake to automate CSSPGO Clang build.

[Amir]

More plumbing

[Amir]

Append to PGO_OPT

[Amir]

perf2prof

[Amir]

Bugfixes

clang/CMakeLists.txt

if(BOOTSTRAP_CMAKE_SYSTEM_NAME STREQUAL "Linux")
       add_dependencies(clang-bootstrap-deps llvm-objcopy llvm-strip)
       set(${CLANG_STAGE}_OBJCOPY -DCMAKE_OBJCOPY=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-objcopy)
       set(${CLANG_STAGE}_STRIP -DCMAKE_STRIP=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-strip)
       set(${CLANG_STAGE}_READELF -DCMAKE_READELF=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-readelf)
     endif()
   endif()
 
   if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED)
+    string(TOUPPER "${BOOTSTRAP_LLVM_BUILD_INSTRUMENTED}"
+    uppercase_BOOTSTRAP_LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps llvm-profdata)
+    if (uppercase_BOOTSTRAP_LLVM_BUILD_INSTRUMENTED STREQUAL "CSSPGO")
+    endif()
     set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata)
   endif()
 
   if(LLVM_BUILD_INSTRUMENTED)
     add_dependencies(clang-bootstrap-deps generate-profdata)
     set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata)
     # Use the current tools for LTO instead of the instrumented ones
     list(APPEND _BOOTSTRAP_DEFAULT_PASSTHROUGH

clang/utils/perf-training/CMakeLists.txt

 set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
   "The path to a lit testsuite containing samples for PGO and order file generation"
   )
 
 if(LLVM_BUILD_INSTRUMENTED)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
     ${CMAKE_CURRENT_BINARY_DIR}/pgo-data/lit.site.cfg
     )
 
   add_lit_testsuite(generate-profraw "Generating clang PGO data"
     ${CMAKE_CURRENT_BINARY_DIR}/pgo-data/
     EXCLUDE_FROM_CHECK_ALL
     DEPENDS clang clear-profraw ${CLANG_PERF_TRAINING_DEPS}
     )
 
+  string(TOUPPER "${LLVM_BUILD_INSTRUMENTED}" uppercase_LLVM_BUILD_INSTRUMENTED)
+  if (uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "CSSPGO")
+    if(NOT LLVM_PROFGEN)
+      find_program(LLVM_PROFGEN llvm-profgen)
+    endif()
+
+    if(NOT LLVM_PROFGEN)
+      message(STATUS "To enable converting CSSPGO samples LLVM_PROFGEN has to point to
+      llvm-profgen")
+    endif()
+
+    # Convert perf profiles into profraw
+    add_custom_target(convert-perf-profraw
+      COMMAND "${Python3_EXECUTABLE}"
+      ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py perf2prof ${LLVM_PROFGEN}
+      $<TARGET_FILE:clang> ${CMAKE_CURRENT_BINARY_DIR}
+  endif()
+
   add_custom_target(clear-profraw
     COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} profraw
     COMMENT "Clearing old profraw data")
 
   if(NOT LLVM_PROFDATA)
     find_program(LLVM_PROFDATA llvm-profdata)
   endif()
 

clang/utils/perf-training/lit.cfg

 sysroot_flags = getSysrootFlagsOnDarwin(config, lit_config)
 
 config.clang = lit.util.which('clang', config.clang_tools_dir).replace('\\', '/')
 
 config.name = 'Clang Perf Training'
 config.suffixes = ['.c', '.cc', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap', '.test']
 
 cc1_wrapper = '%s %s/perf-helper.py cc1' % (config.python_exe, config.perf_helper_dir)
+if config.llvm_build_instrumented.upper() = "CSSPGO":
+    perf_wrapper = "%s %s/perf-helper.py perf --lbr --call-graph -- " % (
+        config.python_exe,
+        config.perf_helper_dir,
+    )
+    cc1_wrapper = perf_wrapper + cc1_wrapper
 
 use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
 config.test_format = lit.formats.ShTest(use_lit_shell == "0")
 config.substitutions.append( ('%clang_cpp_skip_driver', ' %s %s %s ' % (cc1_wrapper, config.clang, sysroot_flags)))
 config.substitutions.append( ('%clang_cpp', ' %s --driver-mode=g++ %s ' % (config.clang, sysroot_flags)))
 config.substitutions.append( ('%clang_skip_driver', ' %s %s %s ' % (cc1_wrapper, config.clang, sysroot_flags)))
 config.substitutions.append( ('%clang', ' %s %s ' % (config.clang, sysroot_flags) ) )
 config.substitutions.append( ('%test_root', config.test_exec_root ) )
 
 config.environment['LLVM_PROFILE_FILE'] = 'perf-training-%4m.profraw'
 

clang/utils/perf-training/lit.site.cfg.in

 @LIT_SITE_CFG_IN_HEADER@
 
 import sys
 
 config.clang_tools_dir = lit_config.substitute("@CURRENT_TOOLS_DIR@")
 config.perf_helper_dir = "@CMAKE_CURRENT_SOURCE_DIR@"
 config.test_exec_root = "@CMAKE_CURRENT_BINARY_DIR@"
 config.test_source_root = "@CLANG_PGO_TRAINING_DATA@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_exe = "@Python3_EXECUTABLE@"
+config.llvm_build_instrumented = "@LLVM_BUILD_INSTRUMENTED@"
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CLANG_SOURCE_DIR@/utils/perf-training/lit.cfg")

clang/utils/perf-training/perf-helper.py

def merge_fdata(args):
     cmd = [args[0], "-o", args[1]]
     cmd.extend(findFilesWithExtension(args[2], "fdata"))
     subprocess.check_call(cmd)
     return 0
 
 
 def perf(args):
     parser = argparse.ArgumentParser(
-        prog="perf-helper perf", description="perf wrapper for BOLT profile collection"
+        prog="perf-helper perf",
+        description="perf wrapper for BOLT/CSSPGO profile collection"
     )
     parser.add_argument(
-        "--lbr", required=False, action="store_true", help="Use perf with branch stacks"
+        "--lbr", required=False, action="store_true",
+        help="Use perf with branch stacks"
+    )
+    parser.add_argument(
+        "--call-graph", required=False, action="store_true",
+        help="Collect call graph"
     )
     parser.add_argument("cmd", nargs="*", help="")
 
     # Use python's arg parser to handle all leading option arguments, but pass
     # everything else through to perf
     first_cmd = next(arg for arg in args if not arg.startswith("--"))
     last_arg_idx = args.index(first_cmd)
 
     opts = parser.parse_args(args[:last_arg_idx])
     # cmd = shlex.split(args[last_arg_idx:])
     cmd = args[last_arg_idx:]
 
     perf_args = []
     perf_args.extend(
         (
             "perf",
             "record",
             "--event=cycles:u",
             "--freq=max",
             "--output=%d.perf.data" % os.getpid(),
         )
     )
     if opts.lbr:
         perf_args += ["--branch-filter=any,u"]
+    if opts.call_graph:
+        perf_args += ["--call-graph=fp"]
     perf_args.extend(cmd)
 
     start_time = time.time()
     subprocess.check_call(perf_args)
 
     elapsed = time.time() - start_time
     print("... data collection took %.4fs" % elapsed)
     return 0
def perf2bolt(args):
     if opts.nolbr:
         p2b_args += ["-nl"]
     p2b_args += ["-p"]
     for filename in findFilesWithExtension(opts.path, "perf.data"):
         subprocess.check_call(p2b_args + [filename, "-o", filename + ".fdata"])
     return 0
 
 
+def perf2prof(args):
+    parser = argparse.ArgumentParser(
+            prog="perf-helper perf2prof",
+            description="perf to CSSPGO prof conversion wrapper",
+            )
+    parser.add_argument("profgen", help="Path to llvm-profgen binary")
+    parser.add_argument("binary", help="Input binary")
+    parser.add_argument("path", help="Path containing perf.data files")
+    opts = parser.parse_args(args)
+
+    perf_args = shlex.split("perf script -F ip,brstack --show-mmap-event -i")
+    profgen_args = [opts.profgen, "--format=text", f"--binary={opts.binary}"]
+    for filename in findFilesWithExtension(opts.path, "perf.data"):
+        perfscript = f"{filename}.perfscript"
+        profraw = f"{filename}.profraw"
+        subprocess.check_call(perf_args + [filename, "-o", perfscript])
+        subprocess.check_call(profgen_args + [f"--perfscript={perfscript}", f"--output={profraw}"])
+    return 0
+
+
 def dtrace(args):
     parser = argparse.ArgumentParser(
         prog="perf-helper dtrace",
         description="dtrace wrapper for order file generation",
     )
     parser.add_argument(
         "--buffer-size",
         metavar="size",

llvm/CMakeLists.txt

else()
   set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default OFF)
 endif()
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ${LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default} CACHE BOOL
   "Enable per-target runtimes directory")
 
 set(LLVM_PROFDATA_FILE "" CACHE FILEPATH
   "Profiling data file to use when compiling in order to improve runtime performance.")
 
+set(LLVM_SPROFDATA_FILE "" CACHE FILEPATH
+  "Sampling profiling data file to use when compiling in order to improve runtime performance.")
+
 if(LLVM_INCLUDE_TESTS)
   # Lit test suite requires at least python 3.6
   set(LLVM_MINIMUM_PYTHON_VERSION 3.6)
 else()
   # FIXME: it is unknown if this is the actual minimum bound
   set(LLVM_MINIMUM_PYTHON_VERSION 3.0)
 endif()
 

llvm/cmake/modules/HandleLLVMOptions.cmake

 option(LLVM_ENABLE_RTTI "Enable run time type information" OFF)
 if(LLVM_ENABLE_EH AND NOT LLVM_ENABLE_RTTI)
   message(FATAL_ERROR "Exception handling requires RTTI. You must set LLVM_ENABLE_RTTI to ON")
 endif()
 
 option(LLVM_ENABLE_IR_PGO "Build LLVM and tools with IR PGO instrumentation (deprecated)" Off)
 mark_as_advanced(LLVM_ENABLE_IR_PGO)
 
-set(LLVM_BUILD_INSTRUMENTED OFF CACHE STRING "Build LLVM and tools with PGO instrumentation. May be specified as IR or Frontend")
+set(LLVM_BUILD_INSTRUMENTED OFF CACHE STRING "Build LLVM and tools with PGO instrumentation. May be specified as IR, Frontend, CSIR, CSSPGO")
 set(LLVM_VP_COUNTERS_PER_SITE "1.5" CACHE STRING "Value profile counters to use per site for IR PGO with Clang")
 mark_as_advanced(LLVM_BUILD_INSTRUMENTED LLVM_VP_COUNTERS_PER_SITE)
 string(TOUPPER "${LLVM_BUILD_INSTRUMENTED}" uppercase_LLVM_BUILD_INSTRUMENTED)
 
 if (LLVM_BUILD_INSTRUMENTED)
   if (LLVM_ENABLE_IR_PGO OR uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "IR")
     append("-fprofile-generate=\"${LLVM_PROFILE_DATA_DIR}\""
       CMAKE_CXX_FLAGS
elseif(uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "CSIR")
     append("-fcs-profile-generate=\"${LLVM_CSPROFILE_DATA_DIR}\""
       CMAKE_CXX_FLAGS
       CMAKE_C_FLAGS)
     if(NOT LINKER_IS_LLD_LINK)
       append("-fcs-profile-generate=\"${LLVM_CSPROFILE_DATA_DIR}\""
         CMAKE_EXE_LINKER_FLAGS
         CMAKE_SHARED_LINKER_FLAGS)
     endif()
+  elseif(uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "CSSPGO")
+    append("-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls -fpseudo-probe-for-profiling"
+      CMAKE_CXX_FLAGS
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+      append("-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls -fpseudo-probe-for-profiling"
+        CMAKE_EXE_LINKER_FLAGS
+        CMAKE_SHARED_LINKER_FLAGS)
+    endif()
   else()
     append("-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\""
       CMAKE_CXX_FLAGS
       CMAKE_C_FLAGS)
     if(NOT LINKER_IS_LLD_LINK)
       append("-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\""
         CMAKE_EXE_LINKER_FLAGS
         CMAKE_SHARED_LINKER_FLAGS)
if(NOT LINKER_IS_LLD_LINK)
         CMAKE_EXE_LINKER_FLAGS
         CMAKE_SHARED_LINKER_FLAGS)
     endif()
   else()
     message(FATAL_ERROR "LLVM_PROFDATA_FILE can only be specified when compiling with clang")
   endif()
 endif()
 
+if(LLVM_SPROFDATA_FILE AND EXISTS ${LLVM_SPROFDATA_FILE})
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
+    append("-fpseudo-probe-for-profiling -fprofile-sample-use=\"${LLVM_SPROFDATA_FILE}\""
+      CMAKE_CXX_FLAGS
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+      append("-fpseudo-probe-for-profiling -fprofile-sample-use=\"${LLVM_SPROFDATA_FILE}\""
+        CMAKE_EXE_LINKER_FLAGS
+        CMAKE_SHARED_LINKER_FLAGS)
+    endif()
+  else()
+    message(FATAL_ERROR "LLVM_SPROFDATA_FILE can only be specified when compiling with clang")
+  endif()
+endif()
+
 option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation" Off)
 mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE)
 append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping"
   CMAKE_CXX_FLAGS
   CMAKE_C_FLAGS
   CMAKE_EXE_LINKER_FLAGS
   CMAKE_SHARED_LINKER_FLAGS)