gn build: Add check-clang-tools to run clang-tools-extra lit tests

[thakis]

Summary

Only runs the clang-tools-extra lit tests; not yet the unit tests.

Add a build file for clangd-indexer too, since it's needed for
the tests.

[mbonadei]

Really nice work!

LGTM (also patched locally and tested).

clang-tools-extra/clangd/indexer/CMakeLists.txt

 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
 
 set(LLVM_LINK_COMPONENTS
     Support
     )
 
 add_clang_executable(clangd-indexer
   IndexerMain.cpp
   )
 
 target_link_libraries(clangd-indexer
   PRIVATE
   clangAST
-  clangIndex
-  clangDaemon
   clangBasic
+  clangDaemon
   clangFrontend
+  clangIndex
   clangLex
   clangTooling
 )

llvm/utils/gn/secondary/BUILD.gn

 import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 
 group("default") {
   deps = [
-    "//clang-tools-extra/clang-apply-replacements/tool:clang-apply-replacements",
-    "//clang-tools-extra/clang-change-namespace/tool:clang-change-namespace",
-    "//clang-tools-extra/clang-doc/tool:clang-doc",
-    "//clang-tools-extra/clang-include-fixer/find-all-symbols/tool:find-all-symbols",
-    "//clang-tools-extra/clang-include-fixer/tool:clang-include-fixer",
-    "//clang-tools-extra/clang-move/tool:clang-move",
-    "//clang-tools-extra/clang-query/tool:clang-query",
-    "//clang-tools-extra/clang-reorder-fields/tool:clang-reorder-fields",
-    "//clang-tools-extra/clang-tidy/tool:clang-tidy",
-    "//clang-tools-extra/clangd/tool:clangd",
-    "//clang-tools-extra/modularize",
-    "//clang-tools-extra/pp-trace",
+    "//clang-tools-extra/test",
     "//clang/test",
     "//lld/test",
     "//llvm/test",
   ]
   if (current_os == "linux" || current_os == "android") {
     deps += [ "//compiler-rt/test/hwasan" ]
   }
 

llvm/utils/gn/secondary/clang-tools-extra/clangd/indexer/BUILD.gn

+executable("clangd-indexer") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clangd",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Index",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "..", ]
+  sources = [
+    "IndexerMain.cpp",
+  ]
+}

llvm/utils/gn/secondary/clang-tools-extra/test/BUILD.gn

+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("clang_tools_extra_lit_site_cfg_files.gni")
+
+template("write_lit_config") {
+  write_cmake_config(target_name) {
+    input = invoker.input
+    output = invoker.output
+    values = [
+      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      "CLANG_TOOLS_BINARY_DIR=" +
+          rebase_path(get_label_info("//clang-tools-extra", "target_out_dir")),
+      "CLANG_TOOLS_SOURCE_DIR=" + rebase_path("//clang-tools-extra"),
+      "LLVM_LIBS_DIR=",  # needed only for shared builds
+      "TARGET_TRIPLE=$llvm_target_triple",
+    ]
+    if (host_os == "win") {
+      # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/bin") ]
+    } else {
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/lib") ]
+    }
+    values += invoker.extra_values
+  }
+}
+
+write_lit_config("lit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang-tools-extra/test/lit.site.cfg.py.in"
+  output = clang_tools_extra_lit_site_cfg_file
+
+  extra_values = [
+    "CLANG_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+    "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+    "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+    "PYTHON_EXECUTABLE=$python_path",
+    "CLANGD_BUILD_XPC_SUPPORT=0",  # FIXME
+  ]
+
+  if (clang_enable_static_analyzer) {
+    extra_values += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
+  } else {
+    extra_values += [ "CLANG_ENABLE_STATIC_ANALYZER=0" ]
+  }
+}
+
+write_lit_config("lit_unit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang-tools-extra/test/Unit/lit.site.cfg.py.in"
+  output = clang_tools_extra_lit_unit_site_cfg_file
+  extra_values = []
+}
+
+# This target should contain all dependencies of check-clang-tools.
+# //:default depends on it, so that ninja's default target builds all
+# prerequisites for check-clang but doesn't run check-clang itself.
+group("test") {
+  deps = [
+    ":lit_site_cfg",
+    ":lit_unit_site_cfg",
+    "//clang-tools-extra/clang-apply-replacements/tool:clang-apply-replacements",
+    "//clang-tools-extra/clang-change-namespace/tool:clang-change-namespace",
+    "//clang-tools-extra/clang-doc/tool:clang-doc",
+    "//clang-tools-extra/clang-include-fixer/find-all-symbols/tool:find-all-symbols",
+    "//clang-tools-extra/clang-include-fixer/tool:clang-include-fixer",
+    "//clang-tools-extra/clang-move/tool:clang-move",
+    "//clang-tools-extra/clang-query/tool:clang-query",
+    "//clang-tools-extra/clang-reorder-fields/tool:clang-reorder-fields",
+    "//clang-tools-extra/clang-tidy/tool:clang-tidy",
+    "//clang-tools-extra/clangd/indexer:clangd-indexer",
+    "//clang-tools-extra/clangd/tool:clangd",
+    "//clang-tools-extra/modularize",
+    "//clang-tools-extra/pp-trace",
+    "//clang/lib/Headers",
+    "//clang/tools/c-index-test",
+    "//clang/tools/clang-rename",
+    "//clang/tools/driver:symlinks",
+    "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/utils/FileCheck",
+    "//llvm/utils/count",
+    "//llvm/utils/llvm-lit",
+    "//llvm/utils/not",
+  ]
+
+  # FIXME: dep on "//clang-tools-extra/unittests" once it exists
+  # FIXME: dep on dexp once it exist
+  testonly = true
+}
+
+action("check-clang-tools") {
+  script = "$root_out_dir/bin/llvm-lit"
+  if (host_os == "win") {
+    script += ".py"
+  }
+  args = [
+    "-sv",
+    "--param",
+    "clang_site_config=" +
+        rebase_path(clang_tools_extra_lit_site_cfg_file, root_out_dir),
+    "--param",
+    "clang_unit_site_config=" +
+        rebase_path(clang_tools_extra_lit_unit_site_cfg_file, root_out_dir),
+    rebase_path(".", root_out_dir),
+  ]
+  outputs = [
+    "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+  ]
+
+  # Since check-clang-tools is always dirty, //:default doesn't depend on it so
+  # that it's not part of the default ninja target.  Hence, check-clang
+  # shouldn't have any deps except :test. so that the default target is sure to
+  # build all the deps.
+  deps = [
+    ":test",
+  ]
+  testonly = true
+
+  pool = "//:console"
+}

llvm/utils/gn/secondary/clang-tools-extra/test/clang_tools_extra_lit_site_cfg_files.gni

+clang_tools_extra_lit_site_cfg_file =
+    "$root_gen_dir/clang-tools-extra/test/lit.site.cfg.py"
+clang_tools_extra_lit_unit_site_cfg_file =
+    "$root_gen_dir/clang-tools-extra/test/Unit/lit.site.cfg.py"

llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn

+import("//clang-tools-extra/test/clang_tools_extra_lit_site_cfg_files.gni")
 import("//clang/test/clang_lit_site_cfg_files.gni")
 import("//lld/test/lld_lit_site_cfg_files.gni")
 import("//llvm/test/llvm_lit_site_cfg_files.gni")
 import("//llvm/utils/gn/build/write_cmake_config.gni")
 
 write_cmake_config("llvm-lit") {
   input = "llvm-lit.in"
   output = "$root_out_dir/bin/llvm-lit"
write_cmake_config("llvm-lit") {
 
   # Generate LLVM_LIT_CONFIG_MAP parameter.
   # llvm-lit contains a mapping from each lit.cfg.py file to the corresponding
   # generated llvm.site.cfg.py file, so llvm-lit depends on all the targets to
   # generate the site.cfg.py file.
   config_map = ""
 
   deps += [
+    "//clang-tools-extra/test:lit_site_cfg",
+    "//clang-tools-extra/test:lit_unit_site_cfg",
     "//clang/test:lit_site_cfg",
     "//clang/test:lit_unit_site_cfg",
     "//lld/test:lit_site_cfg",
     "//lld/test:lit_unit_site_cfg",
     "//llvm/test:lit_site_cfg",
     "//llvm/test:lit_unit_site_cfg",
   ]
 
   # Note: \n is converted into a newline by write_cmake_config.py, not by gn.
+  config_map +=
+      "map_config('" + rebase_path("//clang-tools-extra/test/lit.cfg.py") +
+      "', '" + rebase_path(clang_tools_extra_lit_site_cfg_file) + "')\n"
+  config_map +=
+      "map_config('" + rebase_path("//clang-tools-extra/test/Unit/lit.cfg.py") +
+      "', '" + rebase_path(clang_tools_extra_lit_unit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//clang/test/lit.cfg.py") +
                 "', '" + rebase_path(clang_lit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//clang/test/Unit/lit.cfg.py") +
                 "', '" + rebase_path(clang_lit_unit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//lld/test/lit.cfg.py") + "', '" +
                 rebase_path(lld_lit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//lld/test/Unit/lit.cfg.py") +
                 "', '" + rebase_path(lld_lit_unit_site_cfg_file) + "')\n"