[clang][cli] Benchmark command line round-trip

[jansvoboda11]

Summary

This patch adds a benchmark for command line round-tripping.

Below are the results of running command-line parsing, preprocessing and compilation of a minimal file (int main() { return 0; }) with 137 CC1 arguments (that's typical on macOS; on Linux CC1 usually gets half of that), in release build with assertions:

---------------------------------------------------------------------
Benchmark                              Time           CPU Iterations
---------------------------------------------------------------------
BM_CompilerInvocationCreate/0     145905 ns     145882 ns       4768
BM_CompilerInvocationCreate/1     432513 ns     432354 ns       1622
BM_Preprocess/0                  1442563 ns    1442200 ns        489
BM_Preprocess/1                  1748370 ns    1748310 ns        393
BM_Compile/0                     2656841 ns    2656802 ns        263
BM_Compile/1                     2971966 ns    2970355 ns        231

Command line parsing is ~3x slower. That makes sense given we're doing the parse twice and also generate the original command line from CompilerInvocation. The absolute delta is small though: ~0.3ms. Preprocessing is ~21% slower, compilation ~12% slower.

On a real-world compile of clang/lib/Frontend/CompilerInvocation.cpp with -O3, the command-line parsing time is naturally insignificant:

-----------------------------------------------------------------------
Benchmark                               Time            CPU Iterations 
-----------------------------------------------------------------------
BM_CompilerInvocationCreate/0      208180 ns      208155 ns       3352
BM_CompilerInvocationCreate/1      587957 ns      587869 ns       1183
BM_Preprocess/0                 403769607 ns   403672500 ns          2
BM_Preprocess/1                 405895925 ns   405802000 ns          2
BM_Compile/0                  22408258046 ns 22403926000 ns          1
BM_Compile/1                  22363847808 ns 22358900000 ns          1

Running check-clang and Clang's Frontend LIT tests doesn't show any measurable performance impact.

[jansvoboda11]

clang/benchmarks/CMakeLists.txt

4

Not sure if we should do anything else here. I mostly cargo-culted this from clangd.

clang/benchmarks/CompilerInvocationBench.cpp

12

It would be nice to load this from a file (e.g. ./Inputs/short-args.txt).

What is the best approach here? Some options:

• copy the .txt file to the build folder via CMake and load it using llvm::sys::fs::current_path,
• leave the .txt file in source directory and hard-code the path via CMAKE_CURRENT_SOURCE_DIR,
• require users to supply path to the .txt file through command line.

[jansvoboda11]

Remove hard-coded arguments

[jansvoboda11]

This comment was removed by jansvoboda11.

[jansvoboda11]

Benchmark compilation and preprocessing as well

clang/CMakeLists.txt

 endif()
 add_subdirectory(utils/hmaptool)
 
 if(CLANG_BUILT_STANDALONE)
   llvm_distribution_add_targets()
   process_llvm_pass_plugins()
 endif()
 
+if(LLVM_INCLUDE_BENCHMARKS)
+  add_subdirectory(benchmarks)
+endif()
+
 configure_file(
   ${CLANG_SOURCE_DIR}/include/clang/Config/config.h.cmake
   ${CLANG_BINARY_DIR}/include/clang/Config/config.h)

clang/benchmarks/CMakeLists.txt

+add_benchmark(CompilerInvocationBench
+  CompilerInvocationBench.cpp
+  ${CLANG_SOURCE_DIR}/tools/driver/cc1_main.cpp)
+
+foreach(llvm_target ${LLVM_TARGETS_TO_BUILD})
+  set(target_codegen "LLVM${llvm_target}CodeGen")
+  if(TARGET "${target_codegen}")
+    list(APPEND llvm_target_libraries "${target_codegen}")
+  endif()
+
+  set(target_asm_parser "LLVM${llvm_target}AsmParser")
+  if(TARGET "${target_asm_parser}")
+    list(APPEND llvm_target_libraries "${target_asm_parser}")
+  endif()
+endforeach()
+
+target_link_libraries(CompilerInvocationBench PRIVATE
+  clangBasic
+  clangCodeGen
+  clangDriver
+  clangFrontend
+  clangFrontendTool
+  clangSerialization
+
+  ${llvm_target_libraries}
+  LLVMAnalysis
+  LLVMCodeGen
+  LLVMCore
+#  LLVMIPO
+  LLVMAggressiveInstCombine
+  LLVMInstCombine
+  LLVMInstrumentation
+  LLVMMC
+  LLVMMCParser
+  LLVMObjCARCOpts
+  LLVMOption
+  LLVMScalarOpts
+  LLVMSupport
+  LLVMTarget
+  LLVMTransformUtils
+  LLVMVectorize)

clang/benchmarks/CompilerInvocationBench.cpp

+//===-- CompilerInvocationBench.cpp - Argument parsing benchmark ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/TextDiagnosticBuffer.h"
+
+#include "benchmark/benchmark.h"
+
+using namespace llvm;
+using namespace clang;
+
+int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr);
+
+// After '--' in Argv.
+static const char **Begin;
+
+// The end of Argv.
+static const char **End;
+
+static const char *RoundTrip = "-round-trip-args";
+static const char *NoRoundTrip = "-no-round-trip-args";
+
+static void BM_CompilerInvocationCreate(benchmark::State &State) {
+  SmallVector<const char *, 0> Args{Begin, End};
+  Args.emplace_back(State.range(0) ? RoundTrip : NoRoundTrip);
+
+  auto Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions,
+                                                   new TextDiagnosticBuffer);
+
+  for (auto _ : State) {
+    CompilerInvocation Invocation;
+    CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+    benchmark::DoNotOptimize(Invocation);
+  }
+
+  assert(Diags->getNumErrors() == 0);
+  assert(Diags->getNumWarnings() == 0);
+}
+
+static void BM_Preprocess(benchmark::State &State) {
+  SmallVector<const char *, 0> Args{Begin, End};
+  Args.emplace_back(State.range(0) ? RoundTrip : NoRoundTrip);
+  Args.emplace_back("-E");
+
+  for (auto _ : State) {
+    int ExitCode = cc1_main(Args, "clang", nullptr);
+    benchmark::DoNotOptimize(ExitCode);
+  }
+}
+
+static void BM_Compile(benchmark::State &State) {
+  SmallVector<const char *, 0> Args{Begin, End};
+  Args.emplace_back(State.range(0) ? RoundTrip : NoRoundTrip);
+  Args.emplace_back("-emit-obj");
+
+  for (auto _ : State) {
+    int ExitCode = cc1_main(Args, "clang", nullptr);
+    benchmark::DoNotOptimize(ExitCode);
+  }
+}
+
+BENCHMARK(BM_CompilerInvocationCreate)->Arg(false);
+BENCHMARK(BM_CompilerInvocationCreate)->Arg(true);
+
+BENCHMARK(BM_Preprocess)->Arg(false);
+BENCHMARK(BM_Preprocess)->Arg(true);
+
+BENCHMARK(BM_Compile)->Arg(false);
+BENCHMARK(BM_Compile)->Arg(true);
+
+// USAGE:
+// CompilerInvocationBench [ Google Benchmark arguments ... ] \
+//                      -- [ Clang CC1 arguments ... ]
+int main(int Argc, const char *Argv[]) {
+  // Find the '--' argument.
+  int DashDashIndex = 0;
+  for (int i = 0; i < Argc; ++i)
+    if (StringRef(Argv[i]) == "--")
+      DashDashIndex = i;
+
+  if (DashDashIndex == 0) {
+    llvm::errs() << "USAGE:\n"
+                 << "  " << Argv[0] << " [ Google Benchmark options ... ] -- "
+                 << "[ CompilerInvocation::CreateFromArgs arguments ... ]\n";
+    return 1;
+  }
+
+  Begin = Argv + DashDashIndex + 1;
+  End = Argv + Argc;
+
+  int BenchmarkArgc = DashDashIndex - Argc;
+
+  benchmark::Initialize(&BenchmarkArgc, const_cast<char **>(Argv));
+  benchmark::RunSpecifiedBenchmarks();
+}