diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -14,12 +14,17 @@ ##### Common utilities for update_*test_checks.py +_verbose = False + def parse_commandline_args(parser): parser.add_argument('-v', '--verbose', action='store_true', help='Show verbose output') parser.add_argument('-u', '--update-only', action='store_true', help='Only update test if it was already autogened') - return parser.parse_args() + args = parser.parse_args() + global _verbose + _verbose = args.verbose + return args def should_add_line_to_output(input_line, prefix_set): # Skip any blank comment lines in the IR. @@ -53,7 +58,7 @@ ##### LLVM IR parser -RUN_LINE_RE = re.compile(r'^\s*[;#]\s*RUN:\s*(.*)$') +RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$') CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)') PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$') CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME)?:') @@ -91,6 +96,28 @@ msg = '{}: {}'.format(msg, test_file) print('WARNING: {}'.format(msg), file=sys.stderr) +def debug(*args, **kwargs): + # Python2 does not allow def debug(*args, file=sys.stderr, **kwargs): + if 'file' not in kwargs: + kwargs['file'] = sys.stderr + if _verbose: + print(*args, **kwargs) + +def find_run_lines(test, lines): + debug('Scanning for RUN lines in test file:', test) + raw_lines = [m.group(1) + for m in [RUN_LINE_RE.match(l) for l in lines] if m] + run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] + for l in raw_lines[1:]: + if run_lines[-1].endswith('\\'): + run_lines[-1] = run_lines[-1].rstrip('\\' + ' ' + l) + else: + run_lines.append(l) + debug('Found {} RUN lines in {}:'.format(len(run_lines), test)) + for l in run_lines: + debug(' RUN: {}'.format(l)) + return run_lines + def scrub_body(body): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py --- a/llvm/utils/update_analyze_test_checks.py +++ b/llvm/utils/update_analyze_test_checks.py @@ -69,8 +69,6 @@ test_paths = [test for pattern in args.tests for test in glob.glob(pattern)] for test in test_paths: - if args.verbose: - print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr) with open(test) as f: input_lines = [l.rstrip() for l in f] @@ -84,20 +82,7 @@ common.warn("Skipping test which isn't autogenerated: " + test) continue - raw_lines = [m.group(1) - for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith("\\"): - run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l - else: - run_lines.append(l) - - if args.verbose: - print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr) - for l in run_lines: - print(' RUN: ' + l, file=sys.stderr) - + run_lines = common.find_run_lines(test, input_lines) prefix_list = [] for l in run_lines: if '|' not in l: @@ -132,9 +117,8 @@ for prefix in prefixes: func_dict.update({prefix: dict()}) for prefixes, opt_args in prefix_list: - if args.verbose: - print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr) - print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr) + common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr) + common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr) raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test) @@ -147,8 +131,7 @@ is_in_function = False is_in_function_start = False prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes]) - if args.verbose: - print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr) + common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr) output_lines = [] output_lines.append(autogenerated_note) @@ -194,8 +177,7 @@ continue is_in_function = is_in_function_start = True - if args.verbose: - print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr) + common.debug('Writing %d lines to %s...' % (len(output_lines), test)) with open(test, 'wb') as f: f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines]) diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py --- a/llvm/utils/update_cc_test_checks.py +++ b/llvm/utils/update_cc_test_checks.py @@ -29,7 +29,6 @@ ADVERT = '// NOTE: Assertions have been autogenerated by ' CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:') -RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$') SUBST = { '%clang': [], @@ -38,9 +37,6 @@ } def get_line2spell_and_mangled(args, clang_args): - def debug_mangled(*print_args, **kwargs): - if args.verbose: - print(*print_args, file=sys.stderr, **kwargs) ret = {} # Use clang's JSON AST dump to get the mangled name json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-'] @@ -49,7 +45,7 @@ # -Xclang -ast-dump=json instead: json_dump_args.append('-Xclang') json_dump_args.append('-ast-dump=json') - debug_mangled('Running', ' '.join(json_dump_args)) + common.debug('Running', ' '.join(json_dump_args)) status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if status.returncode != 0: sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n') @@ -67,20 +63,19 @@ if node['kind'] != 'FunctionDecl': continue if node.get('isImplicit') is True and node.get('storageClass') == 'extern': - debug_mangled('Skipping builtin function:', node['name'], '@', node['loc']) + common.debug('Skipping builtin function:', node['name'], '@', node['loc']) continue - debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc']) + common.debug('Found function:', node['kind'], node['name'], '@', node['loc']) line = node['loc'].get('line') # If there is no line it is probably a builtin function -> skip if line is None: - debug_mangled('Skipping function without line number:', node['name'], '@', node['loc']) + common.debug('Skipping function without line number:', node['name'], '@', node['loc']) continue spell = node['name'] mangled = node.get('mangledName', spell) ret[int(line)-1] = (spell, mangled) - if args.verbose: - for line, func_name in sorted(ret.items()): - print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr) + for line, func_name in sorted(ret.items()): + common.debug('line {}: found function {}'.format(line+1, func_name), file=sys.stderr) if not ret: common.warn('Did not find any functions using', ' '.join(json_dump_args)) return ret @@ -191,19 +186,7 @@ continue # Extract RUN lines. - raw_lines = [m.group(1) - for m in [RUN_LINE_RE.match(l) for l in input_lines] if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith("\\"): - run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l - else: - run_lines.append(l) - - if args.verbose: - print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr) - for l in run_lines: - print(' RUN: ' + l, file=sys.stderr) + run_lines = common.find_run_lines(filename, input_lines) # Build a list of clang command lines and check prefixes from RUN lines. run_list = [] @@ -260,9 +243,8 @@ for prefix in prefixes: func_dict.update({prefix: dict()}) for prefixes, clang_args, extra_commands, triple_in_cmd in run_list: - if args.verbose: - print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr) - print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr) + common.debug('Extracted clang cmd: clang {}'.format(clang_args)) + common.debug('Extracted FileCheck prefixes: {}'.format(prefixes)) get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict) diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py --- a/llvm/utils/update_llc_test_checks.py +++ b/llvm/utils/update_llc_test_checks.py @@ -43,8 +43,6 @@ test_paths = [test for pattern in args.tests for test in glob.glob(pattern)] for test in test_paths: - if args.verbose: - print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr) with open(test) as f: input_lines = [l.rstrip() for l in f] @@ -65,20 +63,7 @@ triple_in_ir = m.groups()[0] break - raw_lines = [m.group(1) - for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith("\\"): - run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l - else: - run_lines.append(l) - - if args.verbose: - print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr) - for l in run_lines: - print(' RUN: ' + l, file=sys.stderr) - + run_lines = common.find_run_lines(test, input_lines) run_list = [] for l in run_lines: if '|' not in l: @@ -115,12 +100,6 @@ llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip() if test.endswith('.mir'): llc_cmd_args += ' -x mir' - comment_sym = '#' - check_indent = ' ' - else: - comment_sym = ';' - check_indent = '' - check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd) for item in m.group(1).split(',')] if not check_prefixes: @@ -130,6 +109,12 @@ # now, we just ignore all but the last. run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd)) + if test.endswith('.mir'): + comment_sym = '#' + check_indent = ' ' + else: + comment_sym = ';' + check_indent = '' autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name) func_dict = {} @@ -138,9 +123,8 @@ for prefix in prefixes: func_dict.update({prefix: dict()}) for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list: - if args.verbose: - print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr) - print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr) + common.debug('Extracted LLC cmd:', llc_tool, llc_args) + common.debug('Extracted FileCheck prefixes:', str(prefixes)) raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test) triple = triple_in_cmd or triple_in_ir @@ -154,8 +138,7 @@ is_in_function_start = False func_name = None prefix_set = set([prefix for p in run_list for prefix in p[0]]) - if args.verbose: - print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr) + common.debug('Rewriting FileCheck prefixes:', str(prefix_set)) output_lines = [] output_lines.append(autogenerated_note) @@ -199,8 +182,7 @@ continue is_in_function = is_in_function_start = True - if args.verbose: - print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr) + common.debug('Writing %d lines to %s...' % (len(output_lines), test)) with open(test, 'wb') as f: f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines]) diff --git a/llvm/utils/update_mca_test_checks.py b/llvm/utils/update_mca_test_checks.py --- a/llvm/utils/update_mca_test_checks.py +++ b/llvm/utils/update_mca_test_checks.py @@ -83,26 +83,6 @@ return args -def _find_run_lines(input_lines, args): - raw_lines = [m.group(1) - for m in [common.RUN_LINE_RE.match(l) for l in input_lines] - if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith(r'\\'): - run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l - else: - run_lines.append(l) - - if args.verbose: - sys.stderr.write('Found {} RUN line{}:\n'.format( - len(run_lines), '' if len(run_lines) == 1 else 's')) - for line in run_lines: - sys.stderr.write(' RUN: {}\n'.format(line)) - - return run_lines - - def _get_run_infos(run_lines, args): run_infos = [] for run_line in run_lines: @@ -544,9 +524,7 @@ return sys.stderr.write(' [{} lines total]\n'.format(len(output_lines))) - if args.verbose: - sys.stderr.write( - 'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path)) + common.debug('Writing', len(output_lines), 'lines to', test_path, '..\n\n') with open(test_path, 'wb') as f: f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines]) @@ -562,17 +540,13 @@ # will be written once per source location per test. _configure_warnings(args) - if args.verbose: - sys.stderr.write( - 'Scanning for RUN lines in test file: {}\n'.format(test_path)) - if not os.path.isfile(test_path): raise Error('could not find test file: {}'.format(test_path)) with open(test_path) as f: input_lines = [l.rstrip() for l in f] - run_lines = _find_run_lines(input_lines, args) + run_lines = common.find_run_lines(test_path, input_lines) run_infos = _get_run_infos(run_lines, args) common_prefix, prefix_pad = _get_useful_prefix_info(run_infos) block_infos = _get_block_infos(run_infos, test_path, args, common_prefix) diff --git a/llvm/utils/update_mir_test_checks.py b/llvm/utils/update_mir_test_checks.py --- a/llvm/utils/update_mir_test_checks.py +++ b/llvm/utils/update_mir_test_checks.py @@ -96,22 +96,6 @@ return None -def find_run_lines(test, lines, verbose=False): - raw_lines = [m.group(1) - for m in [common.RUN_LINE_RE.match(l) for l in lines] if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith("\\"): - run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l - else: - run_lines.append(l) - if verbose: - log('Found {} RUN lines:'.format(len(run_lines))) - for l in run_lines: - log(' RUN: {}'.format(l)) - return run_lines - - def build_run_list(test, run_lines, verbose=False): run_list = [] all_prefixes = [] @@ -296,7 +280,6 @@ def update_test_file(args, test): - log('Scanning for RUN lines in test file: {}'.format(test), args.verbose) with open(test) as fd: input_lines = [l.rstrip() for l in fd] @@ -313,7 +296,7 @@ return triple_in_ir = find_triple_in_ir(input_lines, args.verbose) - run_lines = find_run_lines(test, input_lines, args.verbose) + run_lines = common.find_run_lines(test, input_lines) run_list, common_prefixes = build_run_list(test, run_lines, args.verbose) simple_functions = find_functions_with_one_bb(input_lines, args.verbose) diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py --- a/llvm/utils/update_test_checks.py +++ b/llvm/utils/update_test_checks.py @@ -84,8 +84,6 @@ # On Windows we must expand the patterns ourselves. test_paths = [test for pattern in args.tests for test in glob.glob(pattern)] for test in test_paths: - if args.verbose: - print('Scanning for RUN lines in test file: ' + test, file=sys.stderr) with open(test) as f: input_lines = [l.rstrip() for l in f] @@ -99,20 +97,7 @@ common.warn("Skipping test which isn't autogenerated: " + test) continue - raw_lines = [m.group(1) - for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m] - run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] - for l in raw_lines[1:]: - if run_lines[-1].endswith('\\'): - run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l - else: - run_lines.append(l) - - if args.verbose: - print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr) - for l in run_lines: - print(' RUN: ' + l, file=sys.stderr) - + run_lines = common.find_run_lines(test, input_lines) prefix_list = [] for l in run_lines: if '|' not in l: @@ -146,9 +131,8 @@ for prefix in prefixes: func_dict.update({prefix: dict()}) for prefixes, opt_args in prefix_list: - if args.verbose: - print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr) - print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr) + common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args) + common.debug('Extracted FileCheck prefixes: ' + str(prefixes)) raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test) common.build_function_body_dictionary( @@ -159,8 +143,7 @@ is_in_function = False is_in_function_start = False prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes]) - if args.verbose: - print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr) + common.debug('Rewriting FileCheck prefixes:', str(prefix_set)) output_lines = [] output_lines.append(autogenerated_note) @@ -207,8 +190,7 @@ continue is_in_function = is_in_function_start = True - if args.verbose: - print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr) + common.debug('Writing %d lines to %s...' % (len(output_lines), test)) with open(test, 'wb') as f: f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])