[gn build] Add build file for clang/lib/Basic and dependencies

[thakis]

Summary

Adds a build file for clang-tblgen and an action for running it, and uses that to process all the .td files in include/clang/Basic.

Also adds an action to write include/clang/Config/config.h and include/clang/Basic/Version.inc.

[phosek]

LGTM

llvm/trunk/tools/llvm-ar/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   BinaryFormat
   Core
   DlltoolDriver
   LibDriver
   Object
   Support
   )
 
 add_llvm_tool(llvm-ar
   llvm-ar.cpp
-
-  DEPENDS
-  intrinsics_gen
   )
 
 add_llvm_tool_symlink(llvm-ranlib llvm-ar)
 add_llvm_tool_symlink(llvm-lib llvm-ar)
 add_llvm_tool_symlink(llvm-dlltool llvm-ar)
 
 if(LLVM_INSTALL_BINUTILS_SYMLINKS)
   add_llvm_tool_symlink(ar llvm-ar)
   add_llvm_tool_symlink(dlltool llvm-ar)
   add_llvm_tool_symlink(ranlib llvm-ar)
 endif()

llvm/trunk/tools/llvm-bcanalyzer/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   BitReader
   Support
   )
 
 add_llvm_tool(llvm-bcanalyzer
   llvm-bcanalyzer.cpp
-
-  DEPENDS
-  intrinsics_gen
   )

llvm/trunk/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp

 //===--- llvm-demangle-fuzzer.cpp - Fuzzer for the Itanium Demangler ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Demangle/Demangle.h"
 
 #include <cstdint>
 #include <cstdlib>
 #include <string>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
-  std::string NullTerminatedString((const char *)Data, Size);
-  int status = 0;
-  if (char *demangle = llvm::itaniumDemangle(NullTerminatedString.c_str(), nullptr,
-                                         nullptr, &status))
-    free(demangle);
+  if (Size == 0)
+    return 0;
+
+  bool UseItanium = Data[0] < 128;
+  std::string NullTerminatedString((const char *)Data + 1, Size - 1);
+
+  if (UseItanium) {
+    free(llvm::itaniumDemangle(NullTerminatedString.c_str(), nullptr, nullptr,
+                               nullptr));
+  } else {
+    free(llvm::microsoftDemangle(NullTerminatedString.c_str(), nullptr, nullptr,
+                                 nullptr));
+  }
 
   return 0;
 }

llvm/trunk/utils/gn/secondary/BUILD.gn

 group("default") {
   deps = [
     ":lld",
+    "//clang/lib/Basic",
     "//llvm/tools/llc",
     "//llvm/tools/llvm-ar:symlinks",
     "//llvm/tools/llvm-as",
     "//llvm/tools/llvm-bcanalyzer",
     "//llvm/tools/llvm-dis",
     "//llvm/tools/llvm-dwarfdump",
     "//llvm/tools/llvm-mc",
     "//llvm/tools/llvm-nm:symlinks",

llvm/trunk/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn

+import("//clang/utils/TableGen/clang_tablegen.gni")
+import("//llvm/version.gni")
+
+# Version header.
+
+action("version") {
+  script = "//llvm/utils/gn/build/write_cmake_config.py"
+
+  sources = [
+    "Version.inc.in",
+  ]
+  outputs = [
+    "$root_gen_dir/clang/include/clang/Basic/Version.inc",
+  ]
+  args = [
+    "-o",
+    rebase_path(outputs[0], root_out_dir),
+    rebase_path(sources[0], root_out_dir),
+
+    "CLANG_VERSION=$llvm_version",
+    "CLANG_VERSION_MAJOR=$llvm_version_major",
+    "CLANG_VERSION_MINOR=$llvm_version_minor",
+    "CLANG_VERSION_PATCHLEVEL=$llvm_version_patch",
+  ]
+}
+
+# Diagnostics.
+
+diag_groups = [
+  "Analysis",
+  "AST",
+  "Comment",
+  "Common",
+  "CrossTU",
+  "Driver",
+  "Frontend",
+  "Lex",
+  "Parse",
+  "Refactoring",
+  "Sema",
+  "Serialization",
+]
+foreach(diag_group, diag_groups) {
+  clang_tablegen("Diagnostic${diag_group}Kinds") {
+    args = [
+      "-gen-clang-diags-defs",
+      "-clang-component=${diag_group}",
+    ]
+    td_file = "Diagnostic.td"
+  }
+}
+group("diags_tablegen") {
+  # DiagnosticGroups and DiagnosticIndexName are intentionally not part of this
+  # group.  Much of clang depends on the DiagKinds.inc files transitively,
+  # but almost nothing needs DiagnosticGroups.inc or DiagnosticIndexName.inc.
+  public_deps = []
+  foreach(diag_group, diag_groups) {
+    public_deps += [ ":Diagnostic${diag_group}Kinds" ]
+  }
+}
+
+clang_tablegen("DiagnosticGroups") {
+  args = [ "-gen-clang-diag-groups" ]
+  td_file = "Diagnostic.td"
+}
+
+clang_tablegen("DiagnosticIndexName") {
+  args = [ "-gen-clang-diags-index-name" ]
+  td_file = "Diagnostic.td"
+}
+
+# Attributes
+
+clang_tablegen("AttrList") {
+  args = [
+    "-gen-clang-attr-list",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+clang_tablegen("AttrSubMatchRulesList") {
+  args = [
+    "-gen-clang-attr-subject-match-rule-list",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+clang_tablegen("AttrHasAttributeImpl") {
+  args = [
+    "-gen-clang-attr-has-attribute-impl",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+# Misc
+
+clang_tablegen("arm_neon") {
+  args = [ "-gen-arm-neon-sema" ]
+}
+
+clang_tablegen("arm_fp16") {
+  args = [ "-gen-arm-neon-sema" ]
+}

llvm/trunk/utils/gn/secondary/clang/include/clang/Config/BUILD.gn

+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/utils/gn/build/libs/xml/enable.gni")
+import("//llvm/version.gni")
+
+config("Config_config") {
+  visibility = [ ":Config" ]
+  include_dirs = [ "$target_gen_dir/clang/include" ]
+}
+
+action("Config") {
+  script = "//llvm/utils/gn/build/write_cmake_config.py"
+
+  sources = [
+    "config.h.cmake",
+  ]
+  outputs = [
+    "$target_gen_dir/config.h",
+  ]
+  args = [
+    "-o",
+    rebase_path(outputs[0], root_out_dir),
+    rebase_path(sources[0], root_out_dir),
+
+    "BUG_REPORT_URL=https://bugs.llvm.org/",
+    "CLANG_DEFAULT_LINKER=",
+    "CLANG_DEFAULT_STD_C=",
+    "CLANG_DEFAULT_STD_CXX=",
+    "CLANG_DEFAULT_CXX_STDLIB=",
+    "CLANG_DEFAULT_RTLIB=",
+    "CLANG_DEFAULT_OBJCOPY=objcopy",
+    "CLANG_DEFAULT_OPENMP_RUNTIME=libomp",
+    "CLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_35",
+    "CLANG_LIBDIR_SUFFIX=",
+    "CLANG_RESOURCE_DIR=",
+    "C_INCLUDE_DIRS=",
+    "CLANG_CONFIG_FILE_SYSTEM_DIR=",
+    "CLANG_CONFIG_FILE_USER_DIR=",
+    "DEFAULT_SYSROOT=",
+    "GCC_INSTALL_PREFIX=",
+    "CLANG_ANALYZER_WITH_Z3=",
+    "BACKEND_PACKAGE_STRING=LLVM ${llvm_version}svn",
+    "ENABLE_LINKER_BUILD_ID=",
+    "ENABLE_X86_RELAX_RELOCATIONS=",
+    "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=",
+    "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
+  ]
+
+  if (clang_enable_arcmt) {
+    args += [ "CLANG_ENABLE_ARCMT=1" ]
+  } else {
+    args += [ "CLANG_ENABLE_ARCMT=" ]
+  }
+
+  if (clang_enable_static_analyzer) {
+    args += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
+  } else {
+    args += [ "CLANG_ENABLE_STATIC_ANALYZER=" ]
+  }
+
+  if (host_os != "win") {
+    args += [ "CLANG_HAVE_RLIMITS=1" ]
+  } else {
+    args += [ "CLANG_HAVE_RLIMITS=" ]
+  }
+
+  if (llvm_enable_libxml2) {
+    args += [ "CLANG_HAVE_LIBXML=1" ]
+  } else {
+    args += [ "CLANG_HAVE_LIBXML=" ]
+  }
+
+  if (host_os == "mac") {
+    # FIXME: Hardcoding this isn't great, but assuming that the host ld version
+    # has anything to do with the ld version where the built clang will run
+    # isn't either. Probably want to make this a declare_args.
+    args += [ "HOST_LINK_VERSION=305" ]
+  } else {
+    args += [ "HOST_LINK_VERSION=" ]
+  }
+
+  # Let targets depending on this find the generated file.
+  public_configs = [ ":Config_config" ]
+}

llvm/trunk/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni

+declare_args() {
+  # Whether to include the arc migrate tool in the clang binary.
+  clang_enable_arcmt = true
+}

llvm/trunk/utils/gn/secondary/clang/lib/Basic/BUILD.gn

+static_library("Basic") {
+  output_name = "clangBasic"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  public_deps = [
+    # public_dep because public header Version.h includes generated Version.inc.
+    "//clang/include/clang/Basic:AttrList",
+    "//clang/include/clang/Basic:AttrSubMatchRulesList",
+    "//clang/include/clang/Basic:DiagnosticGroups",
+    "//clang/include/clang/Basic:diags_tablegen",
+    "//clang/include/clang/Basic:version",
+  ]
+  deps = [
+    "//clang/include/clang/Basic:AttrHasAttributeImpl",
+    "//clang/include/clang/Basic:arm_fp16",
+    "//clang/include/clang/Basic:arm_neon",
+    "//clang/include/clang/Config",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "Attributes.cpp",
+    "Builtins.cpp",
+    "CharInfo.cpp",
+    "CodeGenOptions.cpp",
+    "Cuda.cpp",
+    "Diagnostic.cpp",
+    "DiagnosticIDs.cpp",
+    "DiagnosticOptions.cpp",
+    "FileManager.cpp",
+    "FileSystemStatCache.cpp",
+    "FixedPoint.cpp",
+    "IdentifierTable.cpp",
+    "LangOptions.cpp",
+    "MemoryBufferCache.cpp",
+    "Module.cpp",
+    "ObjCRuntime.cpp",
+    "OpenMPKinds.cpp",
+    "OperatorPrecedence.cpp",
+    "SanitizerBlacklist.cpp",
+    "SanitizerSpecialCaseList.cpp",
+    "Sanitizers.cpp",
+    "SourceLocation.cpp",
+    "SourceManager.cpp",
+    "TargetInfo.cpp",
+    "Targets.cpp",
+    "Targets/AArch64.cpp",
+    "Targets/AMDGPU.cpp",
+    "Targets/ARC.cpp",
+    "Targets/ARM.cpp",
+    "Targets/AVR.cpp",
+    "Targets/BPF.cpp",
+    "Targets/Hexagon.cpp",
+    "Targets/Lanai.cpp",
+    "Targets/Le64.cpp",
+    "Targets/MSP430.cpp",
+    "Targets/Mips.cpp",
+    "Targets/NVPTX.cpp",
+    "Targets/Nios2.cpp",
+    "Targets/OSTargets.cpp",
+    "Targets/PNaCl.cpp",
+    "Targets/PPC.cpp",
+    "Targets/RISCV.cpp",
+    "Targets/SPIR.cpp",
+    "Targets/Sparc.cpp",
+    "Targets/SystemZ.cpp",
+    "Targets/TCE.cpp",
+    "Targets/WebAssembly.cpp",
+    "Targets/X86.cpp",
+    "Targets/XCore.cpp",
+    "TokenKinds.cpp",
+
+    # FIXME: This should be in its own target that passes -DHAVE_SVN_VERSION_INC
+    # and that also depends on a target generating SVNVersion.inc.
+    "Version.cpp",
+    "Warnings.cpp",
+    "XRayInstr.cpp",
+    "XRayLists.cpp",
+  ]
+}

llvm/trunk/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/enable.gni

+declare_args() {
+  # Whether to include the static analyzer in the clang binary.
+  clang_enable_static_analyzer = true
+}

llvm/trunk/utils/gn/secondary/clang/utils/TableGen/BUILD.gn

+executable("clang-tblgen") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+  ]
+  sources = [
+    "ClangASTNodesEmitter.cpp",
+    "ClangAttrEmitter.cpp",
+    "ClangCommentCommandInfoEmitter.cpp",
+    "ClangCommentHTMLNamedCharacterReferenceEmitter.cpp",
+    "ClangCommentHTMLTagsEmitter.cpp",
+    "ClangDataCollectorsEmitter.cpp",
+    "ClangDiagnosticsEmitter.cpp",
+    "ClangOptionDocEmitter.cpp",
+    "ClangSACheckersEmitter.cpp",
+    "NeonEmitter.cpp",
+    "TableGen.cpp",
+  ]
+}

llvm/trunk/utils/gn/secondary/clang/utils/TableGen/clang_tablegen.gni

+# This file introduces a templates for running clang-tblgen.
+#
+# Parameters:
+#
+#   args (required)
+#       [list of strings] Flags to pass to llvm-tblgen.
+#
+#   output_name (optional)
+#       Basename of the generated output file.
+#       Defaults to target name with ".inc" appended.
+#
+#   td_file (optional)
+#       The .td file to pass to llvm-tblgen.
+#       Defaults to target name with ".td" appended.
+#
+#   visibility (optional)
+#       GN's regular visibility attribute, see `gn help visibility`.
+#
+# Example of usage:
+#
+#   clang_tablegen("DiagnosticGroups") {
+#     args = [ "-gen-clang-diag-groups" ]
+#     td_file = "Diagnostic.td"
+#   }
+
+import("//llvm/utils/TableGen/tablegen.gni")
+
+template("clang_tablegen") {
+  tablegen(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "args",
+                             "output_name",
+                             "td_file",
+                             "visibility",
+                           ])
+
+    # FIXME: In cross builds, this should depend on the host binary.
+    tblgen_target = "//clang/utils/TableGen:clang-tblgen"
+  }
+}

llvm/trunk/utils/gn/secondary/lld/include/lld/Common/BUILD.gn

 import("//llvm/version.gni")
 
 action("version") {
   script = "//llvm/utils/gn/build/write_cmake_config.py"
 
   sources = [
     "Version.inc.in",
   ]
   outputs = [
     "$target_gen_dir/Version.inc",
   ]
   args = [
     "-o",
     rebase_path(outputs[0], root_out_dir),
-
     rebase_path(sources[0], root_out_dir),
 
     "LLD_VERSION=$llvm_version",
     "LLD_VERSION_MAJOR=$llvm_version_major",
     "LLD_VERSION_MINOR=$llvm_version_minor",
 
     # FIXME: Real values for these:
     "LLD_REVISION=",
     "LLD_REPOSITORY=",
   ]
 }

llvm/trunk/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni

template("tablegen") {
   config(config_name) {
     visibility = [ ":$target_name" ]
     include_dirs = [ target_gen_dir ]
   }
 
   action(target_name) {
     forward_variables_from(invoker, [ "visibility" ])
 
+    if (defined(invoker.tblgen_target)) {
+      tblgen_target = invoker.tblgen_target
+    } else {
       # FIXME: In cross builds, this should depend on the host binary.
       tblgen_target = "//llvm/utils/TableGen:llvm-tblgen"
+    }
     tblgen_executable = get_label_info(tblgen_target, "root_out_dir") +
                         "/bin/" + get_label_info(tblgen_target, "name")
     deps = [
       tblgen_target,
     ]
     if (defined(invoker.td_file)) {
       td_file = invoker.td_file
     } else {

llvm/trunk/utils/lit/lit/TestRunner.py

def executeScript(test, litConfig, tmpBase, commands, cwd):
     f.write('\n')
     f.close()
 
     if isWin32CMDEXE:
         command = ['cmd','/c', script]
     else:
         if bashPath:
             command = [bashPath, script]
+            #print command, cwd, test.config.environment
         else:
             command = ['/bin/sh', script]
         if litConfig.useValgrind:
             # FIXME: Running valgrind on sh is overkill. We probably could just
             # run on clang with no real loss.
             command = litConfig.valgrindArgs + command
 
     try:
def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
     if out:
         output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
     if err:
         output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
 
     return lit.Test.Result(status, output)
 
 
-def executeShTest(test, litConfig, useExternalSh,
-                  extra_substitutions=[]):
+def extractScript(test, litConfig, useExternalSh, extra_substitutions):
     if test.config.unsupported:
-        return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported')
+        return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported'), ''
 
     script = parseIntegratedTestScript(test)
     if isinstance(script, lit.Test.Result):
-        return script
+        return script, ''
     if litConfig.noExecute:
-        return lit.Test.Result(Test.PASS)
+        return lit.Test.Result(Test.PASS), ''
 
     tmpDir, tmpBase = getTempPaths(test)
     substitutions = list(extra_substitutions)
     substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
                                              normalize_slashes=useExternalSh)
     script = applySubstitutions(script, substitutions)
+    return script, tmpBase
+
+
+def executeShTest(test, litConfig, useExternalSh,
+                  extra_substitutions=[]):
+    script, tmpBase = extractScript(
+            test, litConfig, useExternalSh, extra_substitutions)
+    if isinstance(script, lit.Test.Result):
+        return script
 
     # Re-run failed tests up to test_retry_attempts times.
     attempts = 1
     if hasattr(test.config, 'test_retry_attempts'):
         attempts += test.config.test_retry_attempts
     for i in range(attempts):
         res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
         if res.code != Test.FAIL:
             break
     # If we had to run the test more than once, count it as a flaky pass. These
     # will be printed separately in the test summary.
     if i > 0 and res.code == Test.PASS:
         res.code = Test.FLAKYPASS
     return res

llvm/trunk/utils/lit/lit/run.py

def __init__(self, lit_config, tests):
         self.parallelism_semaphores = \
                 {k: multiprocessing.Semaphore(v) for k, v in
                  self.lit_config.parallelism_groups.items()}
 
     def execute_test(self, test):
         return _execute_test_impl(test, self.lit_config,
                                   self.parallelism_semaphores)
 
+    def execute_tests_using_ninja(self, jobs, max_time):
+        import lit.TestRunner
+        import pipes
+        sys.path.append('/Users/thakis/src/ninja/misc')
+        import ninja_syntax
+        writer = ninja_syntax.Writer(open('lit.ninja', 'w'))
+        # Writes a build.ninja with commands for all tests and runs ninja
+        # to execute them.
+        for i, test in enumerate(self.tests):
+            if not isinstance(test.config.test_format, lit.formats.ShTest):
+                test.setResult(lit.Test.Result(lit.Test.PASS, ''))  # FIXME
+                continue
+            script, tmpBase = lit.TestRunner.extractScript(
+                    test, self.lit_config, True, [])
+            if isinstance(script, lit.Test.Result):
+                test.result = script
+                continue
+
+            # Create the output directory if it does not already exist.
+            lit.util.mkdir_p(os.path.dirname(tmpBase))
+
+            r = 'r%04d' % i
+            # FIXME: duplication with TestRunner.executeScript()
+            # FIXME: windows
+            execdir = os.path.dirname(test.getExecPath())
+            command = ''
+            for var, val in test.config.environment.iteritems():
+                # Need export, else LLD_VERSION doesn't make it through.
+                # The export means LIT_PRESERVES_TMP=1 needs to be set while
+                # running lit, else it's gone by the time ninja runs
+                # (...or this here must run ninja, probably better anyhoo)
+                command += 'export %s=%s; ' % (var, pipes.quote(val))
+            if test.config.pipefail:
+                command += 'set -o pipefail; '
+            command += 'cd %s; { ' % execdir
+            command += '; } && { '.join(script) + '; }'
+
+            # FIXME: tests require bash for e.g. `echo -e` -- unfortunate :-/
+            command = '/bin/bash -c %s' % pipes.quote(command)
+
+            writer.rule(r, command, description=test.getFullName())
+            writer.build('always%04d' % i, r)
+
+            # FIXME: ...well...
+            test.setResult(lit.Test.Result(lit.Test.PASS, ''))
+
     def execute_tests_in_pool(self, jobs, max_time):
         # We need to issue many wait calls, so compute the final deadline and
         # subtract time.time() from that as we go along.
         deadline = None
         if max_time:
             deadline = time.time() + max_time
 
         # Start a process pool. Copy over the data shared between all test runs.
def execute_tests(self, display, jobs, max_time=None):
         if self.lit_config.singleProcess:
             global child_lit_config
             child_lit_config = self.lit_config
             for test_index, test in enumerate(self.tests):
                 result = worker_run_one_test(test_index, test)
                 self.consume_test_result(result)
         else:
             self.execute_tests_in_pool(jobs, max_time)
+            #self.execute_tests_using_ninja(jobs, max_time)
 
         # Mark any tests that weren't run as UNRESOLVED.
         for test in self.tests:
             if test.result is None:
                 test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
 
     def consume_test_result(self, pool_result):
         """Test completion callback for worker_run_one_test