[update_cc_test_checks.py] Use -ast-dump=json to get mangled name

[arichardson]

Summary

Using c-index-test is fragile since it does not parse all the clang
arguments that are used in the RUN: line. This can result in incorrect
mangled names that do not match any of the generated IR.
For example macOS triples include a leading underscore (which was handled
with a hack in the current script). For the CHERI target we have added
new qualifiers which affect C++ name mangling, but will be included added
by update_cc_test_checks since it parses the source file with the host
triple because it ignores the -triple= argument passed to clang -cc1.

Using the new feature of including the mangled name in the JSON AST dump
(see D69564), we simply parse the output of the RUN: command with
"-fsyntax-only -ast-dump=json" appended.
This should make the script less fragile and also forks one process less.

[MaskRay]

llvm/utils/update_cc_test_checks.py

47

json_dump_args = [args.clang, *clang_args]

48

Can you switch to single quotes? I think that is preferred style and is consistent with other occurrences of string literals.

50

Capitalize for

63

This script is Python 3 only. No need to check sys.version_info

[arichardson]

Address review commments

[MaskRay]

llvm/utils/update_cc_test_checks.py

46

json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']

55

!= 0

Make the type clear

60

Delete the variable output

222

Delete unrelated changes.

232

ditto

[arichardson]

Address comments

llvm/utils/update_cc_test_checks.py

46

I think this syntax requires python 3.5. Is that a reasonable baseline?

For me 3.5 is the oldest version I use this script with. However, that might not be true for everyone?

[arichardson]

Use python 3.5 unpacking syntax

llvm/utils/update_cc_test_checks.py

46

Never mind, subprocess.run() is already python 3.5 only. I'll make that change

[MaskRay]

llvm/utils/update_cc_test_checks.py

269–270

-ast-dump=json -> -Xclang -ast-dump=json

llvm/utils/update_cc_test_checks.py

 #!/usr/bin/env python3
 '''A utility to update LLVM IR CHECK lines in C/C++ FileCheck test files.
 
 Example RUN lines in .c/.cc test files:
 
 // RUN: %clang -emit-llvm -S %s -o - -O2 | FileCheck %s
 // RUN: %clangxx -emit-llvm -S %s -o - -O2 | FileCheck -check-prefix=CHECK-A %s
 
 Usage:
 
 % utils/update_cc_test_checks.py --llvm-bin=release/bin test/a.cc
-% utils/update_cc_test_checks.py --c-index-test=release/bin/c-index-test \
-  --clang=release/bin/clang /tmp/c/a.cc
+% utils/update_cc_test_checks.py --clang=release/bin/clang /tmp/c/a.cc
 '''
 
 import argparse
 import collections
 import distutils.spawn
+import json
 import os
 import shlex
 import string
 import subprocess
 import sys
 import re
 import tempfile
 
 from UpdateTestChecks import asm, common
 
 ADVERT = '// NOTE: Assertions have been autogenerated by '
 
 CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
 RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$')
 
 SUBST = {
     '%clang': [],
     '%clang_cc1': ['-cc1'],
     '%clangxx': ['--driver-mode=g++'],
 }
 
 def get_line2spell_and_mangled(args, clang_args):
+  def debug_mangled(*print_args, **kwargs):
+    if args.verbose:
+      print(*print_args, file=sys.stderr, **kwargs)
   ret = {}
-  with tempfile.NamedTemporaryFile() as f:
-    # TODO Make c-index-test print mangled names without circumventing through precompiled headers
-    status = subprocess.run([args.c_index_test, '-write-pch', f.name, *clang_args],
-                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if status.returncode:
-      sys.stderr.write(status.stdout.decode())
-      sys.exit(2)
-    output = subprocess.check_output([args.c_index_test,
-        '-test-print-mangle', f.name])
-    if sys.version_info[0] > 2:
-      output = output.decode()
-  DeclRE = re.compile(r'^FunctionDecl=(\w+):(\d+):\d+ \(Definition\)')
-  MangleRE = re.compile(r'.*\[mangled=([^]]+)\]')
-  MatchedDecl = False
-  for line in output.splitlines():
-    # Get the function source name, line number and mangled name.  Sometimes
-    # c-index-test outputs the mangled name on a separate line (this can happen
-    # with block comments in front of functions).  Keep scanning until we see
-    # the mangled name.
-    decl_m = DeclRE.match(line)
-    mangle_m = MangleRE.match(line)
-
-    if decl_m:
-      MatchedDecl = True
-      spell, lineno = decl_m.groups()
-    if MatchedDecl and mangle_m:
-      mangled = mangle_m.group(1)
-      MatchedDecl = False
-    else:
-      continue
+  # Use clang's JSON AST dump to get the mangled name
+  json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']
+  if '-cc1' not in json_dump_args:
+    # For tests that invoke %clang instead if %clang_cc1 we have to use
+    # -Xclang -ast-dump=json instead:
+    json_dump_args.append('-Xclang')
+  json_dump_args.append('-ast-dump=json')
+  debug_mangled('Running', ' '.join(json_dump_args))
+  status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  if status.returncode != 0:
+    sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n')
+    sys.stderr.write(status.stderr.decode())
+  ast = json.loads(status.stdout.decode())
+  if ast['kind'] != 'TranslationUnitDecl':
+    common.error('Clang AST dump JSON format changed?')
+    sys.exit(2)
 
-    if mangled == '_' + spell:
-      # HACK for MacOS (where the mangled name includes an _ for C but the IR won't):
-      mangled = spell
-    # Note -test-print-mangle does not print file names so if #include is used,
-    # the line number may come from an included file.
-    ret[int(lineno)-1] = (spell, mangled)
+  # Get the inner node and iterate over all children of type FunctionDecl.
+  # TODO: Should we add checks for global variables being emitted?
+  for node in ast['inner']:
+    if node['kind'] != 'FunctionDecl':
+      continue
+    if node.get('isImplicit') is True and node.get('storageClass') == 'extern':
+      debug_mangled('Skipping builtin function:', node['name'], '@', node['loc'])
+      continue
+    debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc'])
+    line = node['loc'].get('line')
+    # If there is no line it is probably a builtin function -> skip
+    if line is None:
+      debug_mangled('Skipping function without line number:', node['name'], '@', node['loc'])
+      continue
+    spell = node['name']
+    mangled = node.get('mangledName', spell)
+    ret[int(line)-1] = (spell, mangled)
   if args.verbose:
     for line, func_name in sorted(ret.items()):
       print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
+  if not ret:
+    common.warn('Did not find any functions using', ' '.join(json_dump_args))
   return ret
 
 
 def config():
   parser = argparse.ArgumentParser(
       description=__doc__,
       formatter_class=argparse.RawTextHelpFormatter)
   parser.add_argument('-v', '--verbose', action='store_true')
   parser.add_argument('--llvm-bin', help='llvm $prefix/bin path')
   parser.add_argument('--clang',
                       help='"clang" executable, defaults to $llvm_bin/clang')
   parser.add_argument('--clang-args',
                       help='Space-separated extra args to clang, e.g. --clang-args=-v')
-  parser.add_argument('--c-index-test',
-                      help='"c-index-test" executable, defaults to $llvm_bin/c-index-test')
   parser.add_argument('--opt',
                       help='"opt" executable, defaults to $llvm_bin/opt')
   parser.add_argument(
       '--functions', nargs='+', help='A list of function name regexes. '
       'If specified, update CHECK lines for functions matching at least one regex')
   parser.add_argument(
       '--x86_extra_scrub', action='store_true',
       help='Use more regex for x86 matching to reduce diffs between various subtargets')
if args.opt is None:
     else:
       args.opt = os.path.join(args.llvm_bin, 'opt')
   if not distutils.spawn.find_executable(args.opt):
     # Many uses of this tool will not need an opt binary, because it's only
     # needed for updating a test that runs clang | opt | FileCheck. So we
     # defer this error message until we find that opt is actually needed.
     args.opt = None
 
-  if args.c_index_test is None:
-    if args.llvm_bin is None:
-      args.c_index_test = 'c-index-test'
-    else:
-      args.c_index_test = os.path.join(args.llvm_bin, 'c-index-test')
-  if not distutils.spawn.find_executable(args.c_index_test):
-    print('Please specify --llvm-bin or --c-index-test', file=sys.stderr)
-    sys.exit(1)
-
   return args
 
 
 def get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict):
   # TODO Clean up duplication of asm/common build_function_body_dictionary
   # Invoke external tool and extract function bodies.
   raw_tool_output = common.invoke_tool(args.clang, clang_args, filename)
   for extra_command in extra_commands:
for l in run_lines:
 
       triple_in_cmd = None
       m = common.TRIPLE_ARG_RE.search(commands[0])
       if m:
         triple_in_cmd = m.groups()[0]
 
       # Apply %clang substitution rule, replace %s by `filename`, and append args.clang_args
       clang_args = shlex.split(commands[0])
       if clang_args[0] not in SUBST:
         print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)
         continue
       clang_args[0:1] = SUBST[clang_args[0]]
       clang_args = [filename if i == '%s' else i for i in clang_args] + args.clang_args
 
       # Permit piping the output through opt
       if not (len(commands) == 2 or
               (len(commands) == 3 and commands[1].startswith('opt'))):
         print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)
 
       # Extract -check-prefix in FileCheck args
       filecheck_cmd = commands[-1]
       common.verify_filecheck_prefixes(filecheck_cmd)
       if not filecheck_cmd.startswith('FileCheck '):
         print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
         continue
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
for p in run_list:
       for prefix in prefixes:
         func_dict.update({prefix: dict()})
     for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
       if args.verbose:
         print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
         print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)
 
       get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)
 
-      # Invoke c-index-test to get mapping from start lines to mangled names.
-      # Forward all clang args for now.
+      # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to
+      # mangled names. Forward all clang args for now.
       for k, v in get_line2spell_and_mangled(args, clang_args).items():
         line2spell_and_mangled_list[k].append(v)
 
     output_lines = [autogenerated_note]
     for idx, line in enumerate(input_lines):
       # Discard any previous script advertising.
       if line.startswith(ADVERT):
         continue