[gn build] Process .def.in files in llvm/include/Config and that to add a build file for llvm/lib/Target

[thakis]

Summary

Tweak write_cmake_config.py to also handle variable references looking @FOO@ (matching CMake's configure_file() function), and make it replace '\' 'n' in values with a newline literal since there's no good portable way of passing a real newline literal on a command line.

Use that to process all the .def.in files in llvm/include/Config and add llvm/lib/Target, which (indirectly, through llvm-c/Target.h) includes them.

[thakis]

phosek: ping

[phosek]

LGTM

llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn

368

Can you please also provide an error message?

398

Can you put spaces between each target to visually separate them?

llvm/utils/gn/build/write_cmake_config.py

 #!/usr/bin/env python
 """Emulates the bits of CMake's configure_file() function needed in LLVM.
 
 The CMake build uses configure_file() for several things.  This emulates that
-function for the GN build.  In the GN build, this runs at build time, instead
+function for the GN build.  In the GN build, this runs at build time instead
 of at generator time.
 
 Takes a list of KEY=VALUE pairs (where VALUE can be empty).
 
-On each line, replaces ${KEY} with VALUE.
+The sequence `\` `n` in each VALUE is replaced by a newline character.
 
-After that, also handles these special cases (note that FOO= sets the value of
-FOO to the empty string, which is falsy, but FOO=0 sets it to '0' which is
-truthy):
+On each line, replaces '${KEY}' or '@KEY@' with VALUE.
+
+Then, handles these special cases (note that FOO= sets the value of FOO to the
+empty string, which is falsy, but FOO=0 sets it to '0' which is truthy):
 
 1.) #cmakedefine01 FOO
     Checks if key FOO is set to a truthy value, and depending on that prints
     one of the following two lines:
 
         #define FOO 1
         #define FOO 0
 
 2.) #cmakedefine FOO [...]
     Checks if key FOO is set to a truthy in value, and depending on that prints
     one of the following two lines:
 
         #define FOO [...]
         /* #undef FOO */
 
 Fails if any of the KEY=VALUE arguments aren't needed for processing the
-.h.cmake file, or if the .h.cmake file has unreplaced ${VAR} references after
-processing all values.
+input file, or if the input file references keys that weren't passed in.
 """
 
 import argparse
 import os
 import re
 import sys
 
 
 def main():
     parser = argparse.ArgumentParser(
                  epilog=__doc__,
                  formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('input', help='input file')
     parser.add_argument('values', nargs='*', help='several KEY=VALUE pairs')
     parser.add_argument('-o', '--output', required=True,
                         help='output file')
     args = parser.parse_args()
 
     values = {}
     for value in args.values:
         key, val = value.split('=', 1)
-        values[key] = val
+        values[key] = val.replace('\\n', '\n')
     unused_values = set(values.keys())
 
-    # Matches e.g. '${CLANG_FOO}' and captures CLANG_FOO in group 1.
-    var_re = re.compile(r'\$\{([^}]*)\}')
+    # Matches e.g. '${FOO}' or '@FOO@' and captures FOO in group 1 or 2.
+    var_re = re.compile(r'\$\{([^}]*)\}|@([^@]*)@')
 
     in_lines = open(args.input).readlines()
     out_lines = []
     for in_line in in_lines:
         def repl(m):
-            unused_values.discard(m.group(1))
-            return values[m.group(1)]
+            key = m.group(1) or m.group(2)
+            unused_values.discard(key)
+            return values[key]
         in_line = var_re.sub(repl, in_line)
         if in_line.startswith('#cmakedefine01 '):
             _, var = in_line.split()
             in_line = '#define %s %d\n' % (var, 1 if values[var] else 0)
             unused_values.discard(var)
         elif in_line.startswith('#cmakedefine '):
             _, var = in_line.split(None, 1)
             try:
                 var, val = var.split(None, 1)
                 in_line = '#define %s %s' % (var, val)  # val ends in \n.
             except:
                 var = var.rstrip()
                 in_line = '#define %s\n' % var
             if not values[var]:
                 in_line = '/* #undef %s */\n' % var
             unused_values.discard(var)
         out_lines.append(in_line)
 
     if unused_values:
-        print >>sys.stderr, 'Unused --values args:'
+        print >>sys.stderr, 'unused values args:'
         print >>sys.stderr, '    ', '\n    '.join(unused_values)
         return 1
 
     output = ''.join(out_lines)
 
     leftovers = var_re.findall(output)
     if leftovers:
         print >>sys.stderr, 'unprocessed values:\n', '\n'.join(leftovers)
         return 1
 
     if not os.path.exists(args.output) or open(args.output).read() != output:
         open(args.output, 'w').write(output)
 
 
 if __name__ == '__main__':
     sys.exit(main())

llvm/utils/gn/secondary/BUILD.gn

 group("default") {
   deps = [
     "//llvm/lib/Object",
+    "//llvm/lib/Target",
     "//llvm/tools/llvm-undname",
     "//llvm/utils/TableGen:llvm-tblgen",
   ]
 }
 
 # A pool called "console" in the root BUILD.gn is magic and represents ninja's
 # built-in console pool. (Requires a GN with `gn --version` >= 552353.)
 pool("console") {
   depth = 1
 }

llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn

 import("//llvm/lib/Target/targets.gni")
 import("//llvm/triples.gni")
 import("//llvm/utils/gn/build/buildflags.gni")
 import("//llvm/utils/gn/build/libs/pthread/enable.gni")
 import("//llvm/utils/gn/build/libs/terminfo/enable.gni")
 import("//llvm/utils/gn/build/libs/xml/enable.gni")
 import("//llvm/utils/gn/build/libs/zlib/enable.gni")
 import("//llvm/version.gni")
 
-# Contains actions to create config.h, llvm-config.h and abi-breaking.h.
+# Contains actions to create config.h, llvm-config.h, abi-breaking.h,
+# and the various .def files used by llvm/lib/Target.
 # Other than in the cmake build, these are created at build time, not at
 # config time. That way, they can run in parallel, and this pattern means that
 # e.g. creating the clang config header (which happens in another gn file)
 # doesn't block building lld.
 
 # The headers are created by processing the foo.h.cmake files as input,
 # to keep the gn build close to the cmake build.
 
 # Other than in the cmake build, header generation doesn't do any feature
 # checking. See also "Philosophy" in llvm/utils/gn/README.rst.
 
+##############################################################################
+# config.h, llvm-config.h, abi-breaking.h
+
 # FIXME: Several of the config settings go in a global config header but
 # are only used in a single translation unit -- so toggling that value
 # causes a full rebuild when it really only has to rebuild a single file.
 # Instead monolithing config headers, investigate using something like
 # https://is.gd/buildflag_header_gni instead (needs to be done in both the
 # cmake build and here).
 
 # FIXME: This hardcodes a bunch of settings I never use; some of them should
action("llvm-config") {
   }
 
   if (llvm_enable_threads) {
     args += [ "LLVM_ENABLE_THREADS=1" ]
   } else {
     args += [ "LLVM_ENABLE_THREADS=" ]
   }
 }
+
+##############################################################################
+# .def files used by llvm/lib/Target
+
+template("write_target_def_file") {
+  assert(defined(invoker.key))
+  assert(defined(invoker.value))
+
+  action(target_name) {
+    visibility = [ ":write_target_def_files" ]
+    script = "//llvm/utils/gn/build/write_cmake_config.py"
+
+    sources = [
+      "$target_name.in",
+    ]
+    outputs = [
+      "$target_gen_dir/$target_name",
+    ]
+
+    # Build something like
+    # `LLVM_ENUM_ASM_PARSERS=LLVM_ASM_PARSER(ARM)\nLLVM_ASM_PARSER(X86)\n`. Note
+    # that \n is a literal '\' followed by a literal 'n', not a newline
+    # character.  (write_cmake_config.py replaces that with a real newline).
+    value = ""
+    foreach(target, llvm_targets_to_build) {
+      value = "$value${invoker.value}($target)\n"
+    }
+    args = [
+      "-o",
+      rebase_path(outputs[0], root_out_dir),
+      rebase_path(sources[0], root_out_dir),
+      "${invoker.key}=$value",
+    ]
+  }
+}
+write_target_def_file("AsmParsers.def") {
+  key = "LLVM_ENUM_ASM_PARSERS"
+  value = "LLVM_ASM_PARSER"
+}
+write_target_def_file("AsmPrinters.def") {
+  key = "LLVM_ENUM_ASM_PRINTERS"
+  value = "LLVM_ASM_PRINTER"
+}
+write_target_def_file("Disassemblers.def") {
+  key = "LLVM_ENUM_DISASSEMBLERS"
+  value = "LLVM_DISASSEMBLER"
+}
+write_target_def_file("Targets.def") {
+  key = "LLVM_ENUM_TARGETS"
+  value = "LLVM_TARGET"
+}
+group("write_target_def_files") {
+  visibility = [
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  deps = [
+    ":AsmParsers.def",
+    ":AsmPrinters.def",
+    ":Disassemblers.def",
+    ":Targets.def",
+  ]
+}

llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn

static_library("Support") {
   # abi-breaking and llvm-config have been built.
   public_deps = [
     # abi-breaking.h is also include by public headers in ADT, but ADT has
     # no target in the gn build.  Since everything depends on Support, this
     # public_dep does double duty of abi-breaking.h uses in public headers of
     # both Support and ADT.
     "//llvm/include/llvm/Config:abi-breaking",
     "//llvm/include/llvm/Config:llvm-config",
+    # public_dep because public header TargetSelect.h includes these .def files.
+    "//llvm/include/llvm/Config:write_target_def_files",
   ]
   include_dirs = [
     "Unix",
     "Windows",
   ]
   sources = [
     "AArch64TargetParser.cpp",
     "AMDGPUMetadata.cpp",

llvm/utils/gn/secondary/llvm/lib/Target/BUILD.gn

+import("//llvm/lib/Target/targets.gni")
+
+static_library("Target") {
+  output_name = "LLVMTarget"
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  public_deps = [
+    # This is a bit of a hack: llvm-c/Target.h includes llvm/Config/Targets.def,
+    # but there's no target corresponding to llvm-c. Since the functions
+    # declared in llvm-c/Target.h are defined in llvm/lib/Target, clients of
+    # it must depend on llvm/lib/Target, so add the public_dep for Targets.def
+    # here.
+    "//llvm/include/llvm/Config:write_target_def_files",
+  ]
+  sources = [
+    "Target.cpp",
+    "TargetIntrinsicInfo.cpp",
+    "TargetLoweringObjectFile.cpp",
+    "TargetMachine.cpp",
+    "TargetMachineC.cpp",
+  ]
+}