update_test_checks.py: allow use with custom tools

[nhaehnle]

Summary

We have a downstream project with a command-line utility that operates
pretty much exactly like opt. So it would make sense for us to
maintain tests with update_test_checks.py with our custom tool
substituted for opt, as this change allows.

[arsenm]

llvm/test/tools/UpdateTestChecks/update_test_checks/custom-tool.test

1

I can into this problem recently when trying to use llvm-extract. You could use that instead of relying on grep

[nhaehnle]

In the test, use llvm-extract instead of grep

[jsilvanus]

Thanks, I also have a separate possible use case for this. Two minor inline comments.

llvm/utils/update_test_checks.py

10

Maybe update the example to use the new tool argument, and mention above that opt-like
tools are also supported?

122

Maybe also rename opt_args to tool_args, and replace opt by tool in the debug message below?

[nhaehnle]

Address review comments.

[jsilvanus]

LGTM, thanks!

llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/custom-tool.ll

+; Use grep as an easily available "custom tool". Strip out comment lines to get
+; identical output before and after update_test_checks has run.
+; RUN: grep -v ';' %s | FileCheck --check-prefixes=CHECK %s
+
+define i32 @foo(i32 %x) {
+  %y = add i32 %x, 1
+  ret i32 %y
+}

llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/custom-tool.ll.expected

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool grep
+; Use grep as an easily available "custom tool". Strip out comment lines to get
+; identical output before and after update_test_checks has run.
+; RUN: grep -v ';' %s | FileCheck --check-prefixes=CHECK %s
+
+define i32 @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[Y:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = add i32 %x, 1
+  ret i32 %y
+}

llvm/test/tools/UpdateTestChecks/update_test_checks/custom-tool.test

+# Don't rely on 'grep' being available on Windows.
+# UNSUPPORTED: system-windows
+
+# RUN: cp -f %S/Inputs/custom-tool.ll %t.ll && %update_test_checks %t.ll --tool=grep --tool-binary=grep
+# RUN: diff -u %t.ll %S/Inputs/custom-tool.ll.expected
+
+## Check that running the script again does not change the result:
+# RUN: %update_test_checks %t.ll --tool=grep --tool-binary=grep
+# RUN: diff -u %t.ll %S/Inputs/custom-tool.ll.expected

llvm/utils/UpdateTestChecks/common.py

 
 
 def get_autogennote_suffix(parser, args):
   autogenerated_note_args = ''
   for action in parser._actions:
     if not hasattr(args, action.dest):
       continue  # Ignore options such as --help that aren't included in args
     # Ignore parameters such as paths to the binary or the list of tests
-    if action.dest in ('tests', 'update_only', 'opt_binary', 'llc_binary',
-                       'clang', 'opt', 'llvm_bin', 'verbose'):
+    if action.dest in ('tests', 'update_only', 'tool_binary', 'opt_binary',
+                       'llc_binary', 'clang', 'opt', 'llvm_bin', 'verbose'):
       continue
     value = getattr(args, action.dest)
     if action.const is not None:  # action stores a constant (usually True/False)
       # Skip actions with different constant values (this happens with boolean
       # --foo/--no-foo options)
       if value != action.const:
         continue
     if parser.get_default(action.dest) == value:

llvm/utils/update_test_checks.py

 #!/usr/bin/env python3
 
 """A script to generate FileCheck statements for 'opt' regression tests.
 
 This script is a utility to update LLVM opt test cases with new
 FileCheck patterns. It can either update all of the tests in the file or
 a single test function.
 
 Example usage:
 $ update_test_checks.py --opt=../bin/opt test/foo.ll
 
 Workflow:
 1. Make a compiler patch that requires updating some number of FileCheck lines
    in regression test files.
 2. Save the patch and revert it from your local work area.
 3. Update the RUN-lines in the affected regression tests to look canonical.
    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
 4. Refresh the FileCheck lines for either the entire file or select functions by

 import sys
 
 from UpdateTestChecks import common
 
 
 def main():
   from argparse import RawTextHelpFormatter
   parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
-  parser.add_argument('--opt-binary', default='opt',
-                      help='The opt binary used to generate the test case')
+  parser.add_argument('--tool', default='opt',
+                      help='The name of the tool used to generate the test case (defaults to "opt")')
+  parser.add_argument('--tool-binary', '--opt-binary',
+                      help='The tool binary used to generate the test case')
   parser.add_argument(
       '--function', help='The function in the test file to update')
   parser.add_argument('-p', '--preserve-names', action='store_true',
                       help='Do not scrub IR names')
   parser.add_argument('--function-signature', action='store_true',
                       help='Keep function signature information around for the check line')
   parser.add_argument('--scrub-attributes', action='store_true',
                       help='Remove attribute annotations (#0) from the end of check line')
   parser.add_argument('--check-attributes', action='store_true',
                       help='Check "Function Attributes" for functions')
   parser.add_argument('--check-globals', action='store_true',
                       help='Check global entries (global variables, metadata, attribute sets, ...) for functions')
   parser.add_argument('tests', nargs='+')
   initial_args = common.parse_commandline_args(parser)
 
   script_name = os.path.basename(__file__)
-  opt_basename = os.path.basename(initial_args.opt_binary)
-  if not re.match(r'^opt(-\d+)?(\.exe)?$', opt_basename):
-    common.error('Unexpected opt name: ' + opt_basename)
+
+  if initial_args.tool_binary:
+    tool_basename = os.path.basename(initial_args.tool_binary)
+    if not re.match(r'^%s(-\d+)?(\.exe)?$' % (initial_args.tool), tool_basename):
+      common.error('Unexpected tool name: ' + tool_basename)
       sys.exit(1)
-  opt_basename = 'opt'
+  tool_basename = initial_args.tool
 
   for ti in common.itertests(initial_args.tests, parser,
                              script_name='utils/' + script_name):
     # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
     if ti.args.scrub_attributes:
       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
     else:
       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
 
     prefix_list = []
     for l in ti.run_lines:
       if '|' not in l:
         common.warn('Skipping unparseable RUN line: ' + l)
         continue
 
       commands = [cmd.strip() for cmd in l.split('|')]
       assert len(commands) >= 2
       preprocess_cmd = None
       if len(commands) > 2:
         preprocess_cmd = " | ".join(commands[:-2])
       tool_cmd = commands[-2]
       filecheck_cmd = commands[-1]
       common.verify_filecheck_prefixes(filecheck_cmd)
-      if not tool_cmd.startswith(opt_basename + ' '):
-        common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
+      if not tool_cmd.startswith(tool_basename + ' '):
+        common.warn('Skipping non-%s RUN line: %s' % (tool_basename, l))
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
         common.warn('Skipping non-FileChecked RUN line: ' + l)
         continue
 
-      tool_cmd_args = tool_cmd[len(opt_basename):].strip()
+      tool_cmd_args = tool_cmd[len(tool_basename):].strip()
       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 
       check_prefixes = [item for m in
                         common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                         for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
       # FIXME: We should use multiple check prefixes to common check lines. For
       # now, we just ignore all but the last.
       prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 
     global_vars_seen_dict = {}
     builder = common.FunctionTestBuilder(
       run_list=prefix_list,
       flags=ti.args,
       scrubber_args=[],
       path=ti.path)
 
+    tool_binary = ti.args.tool_binary
+    if not tool_binary:
+      tool_binary = tool_basename
+
     for prefixes, opt_args, preprocess_cmd in prefix_list:
-      common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
+      common.debug('Extracted opt cmd: ' + tool_basename + ' ' + opt_args)
       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 
-      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args,
+      raw_tool_output = common.invoke_tool(tool_binary, opt_args,
                                            ti.path, preprocess_cmd=preprocess_cmd,
                                            verbose=ti.args.verbose)
       builder.process_run_line(common.OPT_FUNCTION_RE, common.scrub_body,
               raw_tool_output, prefixes, False)
       builder.processed_prefixes(prefixes)
 
     func_dict = builder.finish_and_get_func_dict()
     is_in_function = False