[gn build] Add check-clang target and make it work

[thakis]

Summary

With this, check-clang runs and passes all of clang's lit tests. It doesn't run any of its unit tests yet.

Like with check-lld, running just ninja -C out/gn will build all prerequisites needed to run tests, but it won't run the tests (so that the build becomes clean after one build). Running ninja -C out/gn check-clang will build prerequisites if needed and run the tests. The check-clang target never becomes clean and runs tests every time.

[thakis]

llvm/utils/gn/secondary/clang/test/BUILD.gn

24

As you can see, this doesn't have all that much in common with llvm/utils/gn/secondary/lld/test/BUILD.gn – it has _some_ stuff in common, but there's also quite a few things different.

[phosek]

llvm/utils/gn/secondary/llvm/triples.gni

10

I don't think host_cpu would ever be missing since it's detected by GN (unless it's a new architecture that GN doesn't recognize), should this rather be unsupported host_cpu or something along those lines?

[thakis]

llvm/utils/gn/secondary/llvm/triples.gni

10

"missing" as in "this is missing code here". "unsupported" doesn't seem quite right since the whole GN build is unsupported :-) And if someone needs a new host_cpu, they can just send a patch. "not-yet-implemented host_cpu " + host_cpu? I don't think the exact error string is super important though.

[thakis]

s/missing/unimplemented/

[phosek]

LGTM

llvm/utils/gn/secondary/BUILD.gn

 import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 
 group("default") {
   deps = [
-    ":clang",
-    "//clang/tools/c-index-test",
-    "//clang/tools/clang-diff",
-    "//clang/tools/clang-format",
-    "//clang/tools/clang-import-test",
-    "//clang/tools/clang-refactor",
-    "//clang/tools/clang-rename",
-    "//clang/tools/diagtool",
-    "//clang/utils/hmaptool",
+    "//clang/test",
     "//lld/test",
-    "//llvm/tools/llvm-cat",
-    "//llvm/tools/llvm-lto",
-    "//llvm/tools/llvm-lto2",
-    "//llvm/tools/llvm-modextract",
-    "//llvm/tools/llvm-profdata",
-    "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/llvm-undname",
   ]
   if (clang_enable_arcmt) {
     deps += [
       "//clang/tools/arcmt-test",
       "//clang/tools/c-arcmt-test",
     ]
   }

llvm/utils/gn/secondary/clang/test/BUILD.gn

+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/lib/Target/targets.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("clang_lit_site_cfg_files.gni")
+
+template("write_lit_config") {
+  action(target_name) {
+    script = "//llvm/utils/gn/build/write_cmake_config.py"
+
+    sources = [
+      invoker.input,
+    ]
+    outputs = [
+      invoker.output,
+    ]
+    args = [
+      "-o",
+      rebase_path(outputs[0], root_out_dir),
+      rebase_path(sources[0], root_out_dir),
+
+      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from ${sources[0]}, do not edit",
+      "CLANG_BINARY_DIR=" +
+          rebase_path(get_label_info("//clang", "target_out_dir")),
+      "CLANG_SOURCE_DIR=" + rebase_path("//clang"),
+      "ENABLE_SHARED=0",
+      "LLVM_BINARY_DIR=" +
+          rebase_path(get_label_info("//llvm", "target_out_dir")),
+      "LLVM_LIBS_DIR=",  # needed only for shared builds
+      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+      "TARGET_TRIPLE=$llvm_target_triple",
+    ]
+    if (host_os == "win") {
+      # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn
+      args += [ "SHLIBDIR=" + rebase_path("$root_out_dir/bin") ]
+    } else {
+      args += [ "SHLIBDIR=" + rebase_path("$root_out_dir/lib") ]
+    }
+    args += invoker.extra_args
+  }
+}
+
+write_lit_config("lit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang/test/lit.site.cfg.py.in"
+  output = clang_lit_site_cfg_file
+
+  extra_args = [
+    "CLANG_ANALYZER_WITH_Z3=",  # Must be empty, not 0.
+    "CLANG_BUILD_EXAMPLES=0",
+    "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
+    "CLANG_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+
+    # This is only used if LLVM_USE_SANITIZER includes lsan and the host
+    # OS is macOS. Since the GN build currently never uses LLVM_USE_SANITIZER,
+    # this is never read.  If it's ever needed,
+    # utils/gn/build/toolchain/BUILD.gn should get the compiler from a variable
+    # that's also read here -- but that should happen after multi-toolchain
+    # builds exist, to make sure it's a toolchain var.
+    "CMAKE_CXX_COMPILER=c++",
+    "ENABLE_BACKTRACES=1",
+    "LLVM_HOST_TRIPLE=$llvm_host_triple",
+    "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+    "LLVM_USE_SANITIZER=",
+    "PYTHON_EXECUTABLE=$python_path",
+    "USE_Z3_SOLVER=",
+  ]
+
+  if (clang_enable_arcmt) {
+    extra_args += [ "CLANG_ENABLE_ARCMT=1" ]
+  } else {
+    extra_args += [ "CLANG_ENABLE_ARCMT=0" ]
+  }
+
+  if (clang_enable_static_analyzer) {
+    extra_args += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
+  } else {
+    extra_args += [ "CLANG_ENABLE_STATIC_ANALYZER=0" ]
+  }
+
+  if (llvm_enable_zlib) {
+    extra_args += [ "HAVE_LIBZ=1" ]
+  } else {
+    extra_args += [ "HAVE_LIBZ=0" ]  # Must be 0.
+  }
+
+  if (host_cpu == "x64") {
+    extra_args += [ "HOST_ARCH=x86_64" ]
+  } else {
+    assert(false, "unimplemented host_cpu " + host_cpu)
+  }
+
+  if (host_os == "mac") {
+    extra_args += [ "LLVM_PLUGIN_EXT=.dylib" ]
+  } else if (host_os == "win") {
+    extra_args += [ "LLVM_PLUGIN_EXT=.dll" ]
+  } else {
+    extra_args += [ "LLVM_PLUGIN_EXT=.so" ]
+  }
+}
+
+write_lit_config("lit_unit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang/test/Unit/lit.site.cfg.py.in"
+  output = clang_lit_unit_site_cfg_file
+  extra_args = [ "LLVM_BUILD_MODE=." ]
+}
+
+# This target should contain all dependencies of check-clang.
+# //:default depends on it, so that ninja's default target builds all
+# prerequisites for check-clang but doesn't run check-clang itself.
+group("test") {
+  deps = [
+    ":lit_site_cfg",
+    ":lit_unit_site_cfg",
+    "//clang/lib/Headers",
+    "//clang/tools/c-index-test",
+    "//clang/tools/clang-diff",
+    "//clang/tools/clang-format",
+    "//clang/tools/clang-import-test",
+    "//clang/tools/clang-offload-bundler",
+    "//clang/tools/clang-refactor",
+    "//clang/tools/clang-rename",
+    "//clang/tools/diagtool",
+    "//clang/tools/driver:symlinks",
+    "//clang/utils/TableGen:clang-tblgen",
+    "//clang/utils/hmaptool",
+    "//llvm/tools/llc",
+    "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/tools/llvm-cat",
+    "//llvm/tools/llvm-config",
+    "//llvm/tools/llvm-dis",
+    "//llvm/tools/llvm-lto",
+    "//llvm/tools/llvm-lto2",
+    "//llvm/tools/llvm-modextract",
+    "//llvm/tools/llvm-nm:symlinks",
+    "//llvm/tools/llvm-objdump:symlinks",
+    "//llvm/tools/llvm-profdata",
+    "//llvm/tools/llvm-readobj:symlinks",
+    "//llvm/tools/llvm-symbolizer:symlinks",
+    "//llvm/tools/opt",
+    "//llvm/utils/FileCheck",
+    "//llvm/utils/count",
+    "//llvm/utils/llvm-lit",
+    "//llvm/utils/not",
+  ]
+  if (clang_enable_arcmt) {
+    deps += [
+      "//clang/tools/arcmt-test",
+      "//clang/tools/c-arcmt-test",
+    ]
+  }
+  if (clang_enable_static_analyzer) {
+    deps += [
+      "//clang/tools/clang-check",
+      "//clang/tools/clang-func-mapping",
+    ]
+  }
+
+  # FIXME: dep on "//clang/unittests" once it exists
+  # FIXME: clang_build_examples
+  testonly = true
+}
+
+action("check-clang") {
+  script = "$root_out_dir/bin/llvm-lit"
+  if (host_os == "win") {
+    script += ".py"
+  }
+  args = [
+    "-sv",
+    "--param",
+    "clang_site_config=" + rebase_path(clang_lit_site_cfg_file, root_out_dir),
+    "--param",
+    "clang_unit_site_config=" +
+        rebase_path(clang_lit_unit_site_cfg_file, root_out_dir),
+    rebase_path(".", root_out_dir),
+  ]
+  outputs = [
+    "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+  ]
+
+  # Since check-clang is always dirty, //:default doesn't depend on it so that
+  # it's not part of the default ninja target.  Hence, check-clang shouldn't
+  # have any deps except :test. so that the default target is sure to build
+  # all the deps.
+  deps = [
+    ":test",
+  ]
+  testonly = true
+
+  pool = "//:console"
+}

llvm/utils/gn/secondary/clang/test/clang_lit_site_cfg_files.gni

+clang_lit_site_cfg_file = "$root_gen_dir/clang/test/lit.site.cfg.py"
+clang_lit_unit_site_cfg_file = "$root_gen_dir/clang/test/Unit/lit.site.cfg.py"

llvm/utils/gn/secondary/llvm/triples.gni

+if (host_cpu == "x64") {
   if (host_os == "linux") {
     llvm_host_triple = "x86_64-unknown-linux-gnu"
   } else if (host_os == "mac") {
     llvm_host_triple = "x86_64-apple-darwin"
   } else if (host_os == "win") {
     llvm_host_triple = "x86_64-pc-windows"
   }
+} else {
+  assert(false, "unimplemented host_cpu " + host_cpu)
+}
 
 declare_args() {
   # The default target triple.
   llvm_target_triple = llvm_host_triple
 }

llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn

+import("//clang/test/clang_lit_site_cfg_files.gni")
 import("//lld/test/lld_lit_site_cfg_files.gni")
 
 action("llvm-lit") {
   script = "//llvm/utils/gn/build/write_cmake_config.py"
 
   sources = [
     "llvm-lit.in",
   ]
action("llvm-lit") {
 
   # Generate LLVM_LIT_CONFIG_MAP parameter.
   # llvm-lit contains a mapping from each lit.cfg.py file to the corresponding
   # generated llvm.site.cfg.py file, so llvm-lit depends on all the targets to
   # generate the site.cfg.py file.
   config_map = ""
 
   deps += [
+    "//clang/test:lit_site_cfg",
+    "//clang/test:lit_unit_site_cfg",
     "//lld/test:lit_site_cfg",
     "//lld/test:lit_unit_site_cfg",
   ]
 
   # Note: \n is converted into a newline by write_cmake_config.py, not by gn.
+  config_map += "map_config('" + rebase_path("//clang/test/lit.cfg.py") +
+                "', '" + rebase_path(clang_lit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//clang/test/Unit/lit.cfg.py") +
+                "', '" + rebase_path(clang_lit_unit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//lld/test/lit.cfg.py") + "', '" +
                 rebase_path(lld_lit_site_cfg_file) + "')\n"
   config_map += "map_config('" + rebase_path("//lld/test/Unit/lit.cfg.py") +
                 "', '" + rebase_path(lld_lit_unit_site_cfg_file) + "')\n"
 
   args = [
     "-o",
     rebase_path(outputs[0], root_out_dir),