[NFC] factor update test function test builder as a class

[mtrofin]

Summary

This allows us to have shared logic over multiple test runs, e.g. do we
have unused prefixes, or which function bodies have conflicting outputs
for a prefix appearing in different RUN lines.

This patch is just wrapping existing functionality, and replacing its uses.
A subsequent patch would then fold the current functionality into the newly
introduced class.

[pengfei]

llvm/utils/UpdateTestChecks/common.py

351

When is close called?

llvm/utils/update_cc_test_checks.py

249–255

I think we can put this code into the class initialization.

251

I think we just need to pass ti.args here. Together with above, it can be simplied to

builder = common.FunctionTestBuilder(run_list, ti.args)

llvm/utils/update_llc_test_checks.py

126

This argument duplicated with class members.

[mtrofin]

llvm/utils/UpdateTestChecks/common.py

351

never - thanks for the catch

llvm/utils/update_cc_test_checks.py

251

that almost worked seamlessly, llc_tests had a different args object, but easy fix. Less unpacking/repacking parameters overall.

llvm/utils/update_llc_test_checks.py

126

not in all cases though. in fact, all but update_llc_test_checks have it []. I'll pass it in the ctor though.

[mtrofin]

update

[pengfei]

LGTM.

llvm/utils/UpdateTestChecks/asm.py

triples = {
       'hexagon': 'hexagon',
   }
   for prefix, triple in triples.items():
     if march.startswith(prefix):
       return triple
   print("Cannot find a triple. Assume 'x86'", file=sys.stderr)
   return 'x86'
 
-def build_function_body_dictionary_for_triple(args, raw_tool_output, triple,
-                                              prefixes, func_dict, func_order):
+def get_run_handler(triple):
   target_handlers = {
       'i686': (scrub_asm_x86, ASM_FUNCTION_X86_RE),
       'x86': (scrub_asm_x86, ASM_FUNCTION_X86_RE),
       'i386': (scrub_asm_x86, ASM_FUNCTION_X86_RE),
       'aarch64': (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_RE),
       'aarch64-apple-darwin': (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
       'hexagon': (scrub_asm_hexagon, ASM_FUNCTION_HEXAGON_RE),
       'r600': (scrub_asm_amdgpu, ASM_FUNCTION_AMDGPU_RE),
def get_run_handler(triple):
   for prefix, s in target_handlers.items():
     if triple.startswith(prefix) and len(prefix) > len(best_prefix):
       handler = s
       best_prefix = prefix
 
   if handler is None:
     raise KeyError('Triple %r is not supported' % (triple))
 
-  scrubber, function_re = handler
-  common.build_function_body_dictionary(
-          function_re, scrubber, [args], raw_tool_output, prefixes,
-          func_dict, func_order, args.verbose, False, False)
+  return handler
 
 ##### Generator of assembly CHECK lines
 
 def add_asm_checks(output_lines, comment_marker, prefix_list, func_dict, func_name):
   # Label format is based on ASM string.
   check_label_format = '{} %s-LABEL: %s%s:'.format(comment_marker)
   global_vars_seen_dict = {}
   common.add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, True, False, global_vars_seen_dict)

llvm/utils/UpdateTestChecks/common.py

for prefix in prefixes:
             # that is different from the one produced by this current RUN line,
             # so the body can't be common accross RUN lines. We use None to
             # indicate that.
             func_dict[prefix][func] = None
             continue
 
       func_dict[prefix][func] = function_body(scrubbed_body, scrubbed_extra, args_and_sig, attrs)
       func_order[prefix].append(func)
-  warn_on_failed_prefixes(func_dict)
+
+class FunctionTestBuilder:
+  def __init__(self, run_list, flags, scrubber_args):
+    self._verbose = flags.verbose
+    self._record_args = flags.function_signature
+    self._check_attributes = flags.check_attributes
+    self._scrubber_args = scrubber_args
+    self._func_dict = {}
+    self._func_order = {}
+    for tuple in run_list:
+      for prefix in tuple[0]:
+        self._func_dict.update({prefix:dict()})
+        self._func_order.update({prefix: []})
+
+  def finish_and_get_func_dict(self):
+    warn_on_failed_prefixes(self._func_dict)
+    return self._func_dict
+
+  def func_order(self):
+    return self._func_order
+  
+  def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes):
+    build_function_body_dictionary(function_re, scrubber, self._scrubber_args,
+                                   raw_tool_output, prefixes, self._func_dict,
+                                   self._func_order, self._verbose,
+                                   self._record_args, self._check_attributes)
 
 ##### Generator of LLVM IR CHECK lines
 
 SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*')
 
 # TODO: We should also derive check lines for global, debug, loop declarations, etc..
 
 class NamelessValue:
     def __init__(self, check_prefix, ir_prefix, ir_regexp):

llvm/utils/update_analyze_test_checks.py

for l in run_lines:
                                for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
       # FIXME: We should use multiple check prefixes to common check lines. For
       # now, we just ignore all but the last.
       prefix_list.append((check_prefixes, tool_cmd_args))
 
-    func_dict = {}
-    func_order = {}
-    for prefixes, _ in prefix_list:
-      for prefix in prefixes:
-        func_dict.update({prefix: dict()})
-        func_order.update({prefix: []})
+    builder = common.FunctionTestBuilder(
+      run_list = prefix_list,
+      flags = args,
+      scrubber_args = [])
+
     for prefixes, opt_args in prefix_list:
       common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
       common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
 
       raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
 
       # Split analysis outputs by "Printing analysis " declarations.
       for raw_tool_output in re.split(r'Printing analysis ', raw_tool_outputs):
-        common.build_function_body_dictionary(
-          common.ANALYZE_FUNCTION_RE, common.scrub_body, [],
-          raw_tool_output, prefixes, func_dict, func_order, args.verbose, False, False)
+        builder.process_run_line(common.ANALYZE_FUNCTION_RE, common.scrub_body,
+                                 raw_tool_output, prefixes)
 
+    func_dict = builder.finish_and_get_func_dict()
     is_in_function = False
     is_in_function_start = False
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
     output_lines = []
     output_lines.append(autogenerated_note)
 
     for input_line in input_lines:

llvm/utils/update_cc_test_checks.py

if not distutils.spawn.find_executable(args.opt):
     # Many uses of this tool will not need an opt binary, because it's only
     # needed for updating a test that runs clang | opt | FileCheck. So we
     # defer this error message until we find that opt is actually needed.
     args.opt = None
 
   return args, parser
 
 
-def get_function_body(args, filename, clang_args, extra_commands, prefixes,
-                      triple_in_cmd, func_dict, func_order):
+def get_function_body(builder, args, filename, clang_args, extra_commands, 
+                      prefixes):
   # TODO Clean up duplication of asm/common build_function_body_dictionary
   # Invoke external tool and extract function bodies.
   raw_tool_output = common.invoke_tool(args.clang, clang_args, filename)
   for extra_command in extra_commands:
     extra_args = shlex.split(extra_command)
     with tempfile.NamedTemporaryFile() as f:
       f.write(raw_tool_output.encode())
       f.flush()
       if extra_args[0] == 'opt':
         if args.opt is None:
           print(filename, 'needs to run opt. '
                 'Please specify --llvm-bin or --opt', file=sys.stderr)
           sys.exit(1)
         extra_args[0] = args.opt
       raw_tool_output = common.invoke_tool(extra_args[0],
                                            extra_args[1:], f.name)
   if '-emit-llvm' in clang_args:
-    common.build_function_body_dictionary(
-            common.OPT_FUNCTION_RE, common.scrub_body, [],
-            raw_tool_output, prefixes, func_dict, func_order, args.verbose,
-            args.function_signature, args.check_attributes)
+    builder.process_run_line(
+            common.OPT_FUNCTION_RE, common.scrub_body, raw_tool_output,
+            prefixes)
   else:
     print('The clang command line should include -emit-llvm as asm tests '
           'are discouraged in Clang testsuite.', file=sys.stderr)
     sys.exit(1)
 
 
 def main():
   initial_args, parser = config()
for l in ti.run_lines:
         print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
         continue
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
       run_list.append((check_prefixes, clang_args, commands[1:-1], triple_in_cmd))
 
     # Execute clang, generate LLVM IR, and extract functions.
-    func_dict = {}
-    func_order = {}
-    for p in run_list:
-      prefixes = p[0]
-      for prefix in prefixes:
-        func_dict.update({prefix: dict()})
-        func_order.update({prefix: []})
+
+    builder = common.FunctionTestBuilder(
+      run_list=run_list,
+      flags=ti.args,
+      scrubber_args=[])
+
     for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
       common.debug('Extracted clang cmd: clang {}'.format(clang_args))
       common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
 
-      get_function_body(ti.args, ti.path, clang_args, extra_commands, prefixes,
-                        triple_in_cmd, func_dict, func_order)
+      get_function_body(builder, ti.args, ti.path, clang_args, extra_commands, 
+                        prefixes)
 
       # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to
       # mangled names. Forward all clang args for now.
       for k, v in get_line2spell_and_mangled(ti.args, clang_args).items():
         line2spell_and_mangled_list[k].append(v)
 
+    func_dict = builder.finish_and_get_func_dict()
     global_vars_seen_dict = {}
     prefix_set = set([prefix for p in run_list for prefix in p[0]])
     output_lines = []
 
     include_generated_funcs = common.find_arg_in_test(ti,
                                                       lambda args: ti.args.include_generated_funcs,
                                                       '--include-generated-funcs',
                                                       True)
if include_generated_funcs:
                                func_dict, func, False,
                                ti.args.function_signature,
                                global_vars_seen_dict)
         else:
           asm.add_asm_checks(my_output_lines, '//',
                              prefixes,
                              func_dict, func)
 
-      common.add_checks_at_end(output_lines, run_list, func_order, '//',
-                               lambda my_output_lines, prefixes, func:
+      common.add_checks_at_end(output_lines, run_list, builder.func_order(), 
+                               '//', lambda my_output_lines, prefixes, func:
                                check_generator(my_output_lines,
                                                prefixes, func))
     else:
       # Normal mode.  Put checks before each source function.
       for line_info in ti.iterlines(output_lines):
         idx = line_info.line_number
         line = line_info.line
         args = line_info.args

llvm/utils/update_llc_test_checks.py

for l in ti.run_lines:
       # now, we just ignore all but the last.
       run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
 
     if ti.path.endswith('.mir'):
       check_indent = '  '
     else:
       check_indent = ''
 
-    func_dict = {}
-    func_order = {}
-    for p in run_list:
-      prefixes = p[0]
-      for prefix in prefixes:
-        func_dict.update({prefix: dict()})
-        func_order.update({prefix: []})
+    builder = common.FunctionTestBuilder(
+        run_list=run_list, 
+        flags=type('', (object,), {
+            'verbose': ti.args.verbose,
+            'function_signature': False,
+            'check_attributes': False}),
+        scrubber_args=[ti.args])
+
     for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
       common.debug('Extracted LLC cmd:', llc_tool, llc_args)
       common.debug('Extracted FileCheck prefixes:', str(prefixes))
 
-      raw_tool_output = common.invoke_tool(ti.args.llc_binary or llc_tool, llc_args, ti.path)
+      raw_tool_output = common.invoke_tool(ti.args.llc_binary or llc_tool,
+                                           llc_args, ti.path)
       triple = triple_in_cmd or triple_in_ir
       if not triple:
         triple = asm.get_triple_from_march(march_in_cmd)
 
-      asm.build_function_body_dictionary_for_triple(ti.args, raw_tool_output,
-          triple, prefixes, func_dict, func_order)
+      scrubber, function_re = asm.get_run_handler(triple)
+      builder.process_run_line(function_re, scrubber, raw_tool_output, prefixes)
+
+    func_dict = builder.finish_and_get_func_dict()
 
     is_in_function = False
     is_in_function_start = False
     func_name = None
     prefix_set = set([prefix for p in run_list for prefix in p[0]])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
 
     include_generated_funcs = common.find_arg_in_test(ti,
                                                       lambda args: ti.args.include_generated_funcs,
                                                       '--include-generated-funcs',
                                                       True)
 
     if include_generated_funcs:
       # Generate the appropriate checks for each function.  We need to emit
       # these in the order according to the generated output so that CHECK-LABEL
       # works properly.  func_order provides that.
 
       # We can't predict where various passes might insert functions so we can't
       # be sure the input function order is maintained.  Therefore, first spit
       # out all the source lines.
       common.dump_input_lines(output_lines, ti, prefix_set, ';')
 
       # Now generate all the checks.
-      common.add_checks_at_end(output_lines, run_list, func_order,
+      common.add_checks_at_end(output_lines, run_list, builder.func_order(),
                                check_indent + ';',
                                lambda my_output_lines, prefixes, func:
                                asm.add_asm_checks(my_output_lines,
                                                   check_indent + ';',
                                                   prefixes, func_dict, func))
     else:
       for input_info in ti.iterlines(output_lines):
         input_line = input_info.line

llvm/utils/update_test_checks.py

for l in ti.run_lines:
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
       # FIXME: We should use multiple check prefixes to common check lines. For
       # now, we just ignore all but the last.
       prefix_list.append((check_prefixes, tool_cmd_args))
 
     global_vars_seen_dict = {}
-    func_dict = {}
-    func_order = {}
-    for prefixes, _ in prefix_list:
-      for prefix in prefixes:
-        func_dict.update({prefix: dict()})
-        func_order.update({prefix: []})
+    builder = common.FunctionTestBuilder(
+      run_list=prefix_list,
+      flags=ti.args,
+      scrubber_args=[])
+
     for prefixes, opt_args in prefix_list:
       common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 
-      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args, ti.path)
-      common.build_function_body_dictionary(
-              common.OPT_FUNCTION_RE, common.scrub_body, [],
-              raw_tool_output, prefixes, func_dict, func_order, ti.args.verbose,
-              ti.args.function_signature, ti.args.check_attributes)
+      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args, 
+                                           ti.path)
+      builder.process_run_line(common.OPT_FUNCTION_RE, common.scrub_body,
+              raw_tool_output, prefixes)
 
+    func_dict = builder.finish_and_get_func_dict()
     is_in_function = False
     is_in_function_start = False
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
 
     include_generated_funcs = common.find_arg_in_test(ti,
                                                       lambda args: ti.args.include_generated_funcs,
                                                       '--include-generated-funcs',
                                                       True)
 
     if include_generated_funcs:
       # Generate the appropriate checks for each function.  We need to emit
       # these in the order according to the generated output so that CHECK-LABEL
       # works properly.  func_order provides that.
 
       # We can't predict where various passes might insert functions so we can't
       # be sure the input function order is maintained.  Therefore, first spit
       # out all the source lines.
       common.dump_input_lines(output_lines, ti, prefix_set, ';')
 
       # Now generate all the checks.
-      common.add_checks_at_end(output_lines, prefix_list, func_order, ';',
-                               lambda my_output_lines, prefixes, func:
+      common.add_checks_at_end(output_lines, prefix_list, builder.func_order(),
+                               ';', lambda my_output_lines, prefixes, func:
                                common.add_ir_checks(my_output_lines, ';',
                                                     prefixes,
                                                     func_dict, func, False,
                                                     ti.args.function_signature,
                                                     global_vars_seen_dict))
     else:
       # "Normal" mode.
       for input_line_info in ti.iterlines(output_lines):