Index: utils/UpdateTestChecks/common.py =================================================================== --- /dev/null +++ utils/UpdateTestChecks/common.py @@ -0,0 +1,61 @@ +import re +import subprocess + +RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$') +CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?=(\S+)') +CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:') + +SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)') +SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) +SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) +SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') +SCRUB_LOOP_COMMENT_RE = re.compile( + r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M) + +def should_add_line_to_output(input_line, prefix_set): + # Skip any blank comment lines in the IR. + if input_line.strip() == ';': + return False + # Skip any blank lines in the IR. + #if input_line.strip() == '': + # return False + # And skip any CHECK lines. We're building our own. + m = CHECK_RE.match(input_line) + if m and m.group(1) in prefix_set: + return False + + return True + +# Invoke the tool that is being tested. +def invoke_tool(exe, cmd_args, ir): + with open(ir) as ir_file: + stdout = subprocess.check_output(exe + ' ' + cmd_args, + shell=True, stdin=ir_file) + # Fix line endings to unix CR style. + stdout = stdout.replace('\r\n', '\n') + return stdout + +# Build up a dictionary of all the function bodies. +def build_function_body_dictionary(function_re, scrubber, scrubber_args, raw_tool_output, prefixes, func_dict, verbose): + for m in function_re.finditer(raw_tool_output): + if not m: + continue + func = m.group('func') + scrubbed_body = scrubber(m.group('body'), *scrubber_args) + if func.startswith('stress'): + # We only use the last line of the function body for stress tests. + scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:]) + if verbose: + print >>sys.stderr, 'Processing function: ' + func + for l in scrubbed_body.splitlines(): + print >>sys.stderr, ' ' + l + for prefix in prefixes: + if func in func_dict[prefix] and func_dict[prefix][func] != scrubbed_body: + if prefix == prefixes[-1]: + print >>sys.stderr, ('WARNING: Found conflicting asm under the ' + 'same prefix: %r!' % (prefix,)) + else: + func_dict[prefix][func] = None + continue + + func_dict[prefix][func] = scrubbed_body Index: utils/update_llc_test_checks.py =================================================================== --- utils/update_llc_test_checks.py +++ utils/update_llc_test_checks.py @@ -14,15 +14,7 @@ import sys import re -# Invoke the tool that is being tested. -def llc(args, cmd_args, ir): - with open(ir) as ir_file: - stdout = subprocess.check_output(args.llc_binary + ' ' + cmd_args, - shell=True, stdin=ir_file) - # Fix line endings to unix CR style. - stdout = stdout.replace('\r\n', '\n') - return stdout - +from UpdateTestChecks import common # RegEx: this is where the magic happens. @@ -81,9 +73,9 @@ flags=(re.M | re.S)) -SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) -SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) -SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') +common.SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) +common.SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) +common.SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') SCRUB_LOOP_COMMENT_RE = re.compile( r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M) @@ -96,17 +88,14 @@ SCRUB_X86_LCP_RE = re.compile(r'\.LCPI[0-9]+_[0-9]+') SCRUB_X86_RET_RE = re.compile(r'ret[l|q]') -RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$') TRIPLE_ARG_RE = re.compile(r'-mtriple=([^ ]+)') TRIPLE_IR_RE = re.compile(r'^target\s+triple\s*=\s*"([^"]+)"$') IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@(\w+)\s*\(') -CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?=(\S+)') -CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:') def scrub_asm_x86(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Detect shuffle asm comments and hide the operands in favor of the comments. @@ -121,69 +110,67 @@ # Avoid generating different checks for 32- and 64-bit because of 'retl' vs 'retq'. asm = SCRUB_X86_RET_RE.sub(r'ret{{[l|q]}}', asm) # Strip kill operands inserted into the asm. - asm = SCRUB_KILL_COMMENT_RE.sub('', asm) + asm = common.SCRUB_KILL_COMMENT_RE.sub('', asm) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm def scrub_asm_arm_eabi(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Strip kill operands inserted into the asm. - asm = SCRUB_KILL_COMMENT_RE.sub('', asm) + asm = common.SCRUB_KILL_COMMENT_RE.sub('', asm) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm def scrub_asm_powerpc64(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Stripe unimportant comments asm = SCRUB_LOOP_COMMENT_RE.sub(r'', asm) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm def scrub_asm_mips(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm def scrub_asm_riscv(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm def scrub_asm_systemz(asm, args): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - asm = SCRUB_WHITESPACE_RE.sub(r' ', asm) + asm = common.SCRUB_WHITESPACE_RE.sub(r' ', asm) # Expand the tabs used for indentation. asm = string.expandtabs(asm, 2) # Strip trailing whitespace. - asm = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) + asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm) return asm -# Build up a dictionary of all the function bodies. -def build_function_body_dictionary(raw_tool_output, triple, prefixes, func_dict, - args): +def build_function_body_dictionary1(args, raw_tool_output, triple, prefixes, func_dict): target_handlers = { 'x86_64': (scrub_asm_x86, ASM_FUNCTION_X86_RE), 'i686': (scrub_asm_x86, ASM_FUNCTION_X86_RE), @@ -227,28 +214,10 @@ raise KeyError('Triple %r is not supported' % (triple)) scrubber, function_re = handlers - for m in function_re.finditer(raw_tool_output): - if not m: - continue - func = m.group('func') - scrubbed_body = scrubber(m.group('body'), args) - if func.startswith('stress'): - # We only use the last line of the function body for stress tests. - scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:]) - if args.verbose: - print >>sys.stderr, 'Processing function: ' + func - for l in scrubbed_body.splitlines(): - print >>sys.stderr, ' ' + l - for prefix in prefixes: - if func in func_dict[prefix] and func_dict[prefix][func] != scrubbed_body: - if prefix == prefixes[-1]: - print >>sys.stderr, ('WARNING: Found conflicting asm under the ' - 'same prefix: %r!' % (prefix,)) - else: - func_dict[prefix][func] = None - continue + common.build_function_body_dictionary( + function_re, scrubber, [args], raw_tool_output, prefixes, + func_dict, args.verbose) - func_dict[prefix][func] = scrubbed_body def add_checks(output_lines, run_list, func_dict, func_name): @@ -275,20 +244,6 @@ return output_lines -def should_add_line_to_output(input_line, prefix_set): - # Skip any blank comment lines in the IR. - if input_line.strip() == ';': - return False - # Skip any blank lines in the IR. - #if input_line.strip() == '': - # return False - # And skip any CHECK lines. We're building our own. - m = CHECK_RE.match(input_line) - if m and m.group(1) in prefix_set: - return False - - return True - def main(): parser = argparse.ArgumentParser(description=__doc__) @@ -321,7 +276,7 @@ break raw_lines = [m.group(1) - for m in [RUN_LINE_RE.match(l) for l in input_lines] if m] + for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m] run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] for l in raw_lines[1:]: if run_lines[-1].endswith("\\"): @@ -358,7 +313,7 @@ llc_cmd_args = llc_cmd[len('llc'):].strip() llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip() - check_prefixes = [item for m in CHECK_PREFIX_RE.finditer(filecheck_cmd) + check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd) for item in m.group(1).split(',')] if not check_prefixes: check_prefixes = ['CHECK'] @@ -377,12 +332,12 @@ print >>sys.stderr, 'Extracted LLC cmd: llc ' + llc_args print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes) - raw_tool_output = llc(args, llc_args, test) + raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test) if not (triple_in_cmd or triple_in_ir): print >>sys.stderr, "Cannot find a triple. Assume 'x86'" - build_function_body_dictionary(raw_tool_output, - triple_in_cmd or triple_in_ir or 'x86', prefixes, func_dict, args) + build_function_body_dictionary1(args, raw_tool_output, + triple_in_cmd or triple_in_ir or 'x86', prefixes, func_dict) is_in_function = False is_in_function_start = False @@ -398,7 +353,7 @@ if input_line == '': continue if input_line.lstrip().startswith(';'): - m = CHECK_RE.match(input_line) + m = common.CHECK_RE.match(input_line) if not m or m.group(1) not in prefix_set: output_lines.append(input_line) continue @@ -408,7 +363,7 @@ is_in_function_start = False if is_in_function: - if should_add_line_to_output(input_line, prefix_set) == True: + if common.should_add_line_to_output(input_line, prefix_set) == True: # This input line of the function body will go as-is into the output. output_lines.append(input_line) else: Index: utils/update_test_checks.py =================================================================== --- utils/update_test_checks.py +++ utils/update_test_checks.py @@ -38,76 +38,36 @@ import tempfile import re +from UpdateTestChecks import common + ADVERT = '; NOTE: Assertions have been autogenerated by ' # RegEx: this is where the magic happens. -SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)') -SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) -SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) -SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*') -RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$') IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(') OPT_FUNCTION_RE = re.compile( r'^\s*define\s+(?:internal\s+)?[^@]*@(?P[\w-]+?)\s*\(' r'(\s+)?[^)]*[^{]*\{\n(?P.*?)^\}$', flags=(re.M | re.S)) -CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?=(\S+)') -CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:') # Match things that look at identifiers, but only if they are followed by # spaces, commas, paren, or end of the string IR_VALUE_RE = re.compile(r'(\s+)%([\w\.]+?)([,\s\(\)]|\Z)') -# Invoke the tool that is being tested. -def invoke_tool(args, cmd_args, ir): - with open(ir) as ir_file: - stdout = subprocess.check_output(args.opt_binary + ' ' + cmd_args, - shell=True, stdin=ir_file) - # Fix line endings to unix CR style. - stdout = stdout.replace('\r\n', '\n') - return stdout - def scrub_body(body, opt_basename): # Scrub runs of whitespace out of the assembly, but leave the leading # whitespace in place. - body = SCRUB_WHITESPACE_RE.sub(r' ', body) + body = common.SCRUB_WHITESPACE_RE.sub(r' ', body) # Expand the tabs used for indentation. body = string.expandtabs(body, 2) # Strip trailing whitespace. - body = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', body) + body = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', body) return body -# Build up a dictionary of all the function bodies. -def build_function_body_dictionary(raw_tool_output, prefixes, func_dict, verbose, opt_basename): - func_regex = OPT_FUNCTION_RE - for m in func_regex.finditer(raw_tool_output): - if not m: - continue - func = m.group('func') - scrubbed_body = scrub_body(m.group('body'), opt_basename) - if func.startswith('stress'): - # We only use the last line of the function body for stress tests. - scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:]) - if verbose: - print >>sys.stderr, 'Processing function: ' + func - for l in scrubbed_body.splitlines(): - print >>sys.stderr, ' ' + l - for prefix in prefixes: - if func in func_dict[prefix] and func_dict[prefix][func] != scrubbed_body: - if prefix == prefixes[-1]: - print >>sys.stderr, ('WARNING: Found conflicting asm under the ' - 'same prefix: %r!' % (prefix,)) - else: - func_dict[prefix][func] = None - continue - - func_dict[prefix][func] = scrubbed_body - # Create a FileCheck variable name based on an IR name. def get_value_name(var): @@ -213,21 +173,6 @@ return output_lines -def should_add_line_to_output(input_line, prefix_set): - # Skip any blank comment lines in the IR. - if input_line.strip() == ';': - return False - # Skip any blank lines in the IR. - #if input_line.strip() == '': - # return False - # And skip any CHECK lines. We're building our own. - m = CHECK_RE.match(input_line) - if m and m.group(1) in prefix_set: - return False - - return True - - def main(): from argparse import RawTextHelpFormatter parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) @@ -254,7 +199,7 @@ input_lines = [l.rstrip() for l in f] raw_lines = [m.group(1) - for m in [RUN_LINE_RE.match(l) for l in input_lines] if m] + for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m] run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] for l in raw_lines[1:]: if run_lines[-1].endswith("\\"): @@ -282,7 +227,7 @@ tool_cmd_args = tool_cmd[len(opt_basename):].strip() tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip() - check_prefixes = [item for m in CHECK_PREFIX_RE.finditer(filecheck_cmd) + check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd) for item in m.group(1).split(',')] if not check_prefixes: check_prefixes = ['CHECK'] @@ -300,8 +245,10 @@ print >>sys.stderr, 'Extracted opt cmd: ' + opt_basename + ' ' + opt_args print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes) - raw_tool_output = invoke_tool(args, opt_args, test) - build_function_body_dictionary(raw_tool_output, prefixes, func_dict, args.verbose, opt_basename) + raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test) + common.build_function_body_dictionary( + OPT_FUNCTION_RE, scrub_body, [opt_basename], raw_tool_output, + prefixes, func_dict, args.verbose) is_in_function = False is_in_function_start = False @@ -316,7 +263,7 @@ if input_line == '': continue if input_line.lstrip().startswith(';'): - m = CHECK_RE.match(input_line) + m = common.CHECK_RE.match(input_line) if not m or m.group(1) not in prefix_set: output_lines.append(input_line) continue @@ -326,10 +273,10 @@ is_in_function_start = False if is_in_function: - if should_add_line_to_output(input_line, prefix_set) == True: + if common.should_add_line_to_output(input_line, prefix_set) == True: # This input line of the function body will go as-is into the output. # Except make leading whitespace uniform: 2 spaces. - input_line = SCRUB_LEADING_WHITESPACE_RE.sub(r' ', input_line) + input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r' ', input_line) output_lines.append(input_line) else: continue