[Utils][WIP] Refactor script for cc tests

[ggeorgakoudis]

Update regex for removing old checklines

[ggeorgakoudis]

Update fopenmp-simd handling to avoid duplicate implicit checks

[ggeorgakoudis]

Update correction for implicit simd checks

clang/utils/refactor_cc_tests.py

+#!/usr/bin/env python3
+
+import argparse
+import re
+import subprocess
+import sys
+import multiprocessing
+from itertools import repeat
+from itertools import starmap
+
+def run_update_cc(filename, llvm_util_dir, llvm_bin):
+    # Probably we don't need to those arguments since we added a NOTE line.
+    update_cc = [ llvm_util_dir + '/update_cc_test_checks.py', '--llvm-bin', llvm_bin,
+            '--include-generated-func',
+            '--function-signature',
+            '--replace-value-regex', '\"__omp_offloading_[0-9a-z]+_[0-9a-z]+\"',
+            '\"reduction_size[.].+[.]\"',
+            '\"pl_cond[.].+[.|,]\"',
+            '--prefix-filecheck-ir-name', '_',
+            filename]
+
+    # Run update_cc_test_checks.py.
+    print('Running update_cc')
+    popen = subprocess.Popen(update_cc, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    stdout, stderr = popen.communicate()
+    if popen.returncode != 0:
+        sys.stderr.write('Failed to run update_cc '+ ' '.join(update_cc) + '\n')
+        sys.stderr.write(stderr)
+        sys.stderr.write(stdout)
+        return popen.returncode
+
+    return popen.returncode
+
+def refactor(filename, llvm_util_dir, llvm_bin):
+    print('Processing file ', filename, '...')
+    with open(filename, 'r') as f:
+        input_lines = f.readlines()
+        output_lines = ''
+        cnt = 1
+        check_prefix_set = {'CHECK'}
+        for line in input_lines:
+            # Modify run line.
+            if line.startswith('// RUN'):
+                # Process FileCheck in line.
+                if 'FileCheck' in line:
+                    m = re.findall(r'--?check-prefix(?:es)?(?:\s|=)(.+?)\s+', line)
+                    # Extract check prefixes to remove check lines.
+                    check_prefix_list = []
+                    for i in m:
+                        check_prefix_list += i.split(',')
+                    check_prefix_set.update(set(check_prefix_list))
+                    # Delete all check prefixes.
+                    if m:
+                        line = re.sub(r'(?:-|--)check-prefix(?:es|)(?:\s|=).+?\s+', r'', line)
+
+                    # Generate implicit check.
+                    if '-fopenmp-simd' in line and \
+                    not re.search(r'\b-fopenmp\b', line) and \
+                    not re.search(r' simd', ''.join(input_lines)):
+                        if not re.search(r'--implicit-check-not="{{__kmpc|__tgt}}"', line):
+                            line = line[:-1] + \
+                                ' --implicit-check-not="{{__kmpc|__tgt}}"\n'
+                    # Generate check prefix using the counter.
+                    else:
+                        line = line[:-1] + ' --check-prefix=CHECK%d\n'%(cnt)
+                        cnt += 1
+
+                # Replace generic triple.
+                # TODO: add ms_abi_triple
+                if '%itanium_abi_triple' in line:
+                    line = re.sub(r'%itanium_abi_triple', 'x86_64-unknown-linux', line)
+
+                if 'clang' in line and not '-triple' in line:
+                    line = re.sub(r'clang(_cc1|xx)?\s+', r'clang\g<1> -triple x86_64-unknown-linux ', line)
+                output_lines += line
+            elif line.startswith('// NOTE:'):
+                # Skip NOTE line, script will add its own.
+                continue
+            else:
+                check_prefix_regex = '(' + '|'.join(check_prefix_set) + ')'
+                # Discard check line.
+                if re.match(r'\s*//\s*%s'%(check_prefix_regex), line):
+                    continue
+
+                # Skip empty comment lines to match update_cc handling.
+                line_strip = line.strip()
+                if line_strip == '//':
+                    pass
+                # Keep line.
+                else:
+                    output_lines += line
+    # Add NOTE to avoid omp_offloading line number problems
+    NOTE = ('// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py'
+    ' UTC_ARGS: --function-signature --include-generated-funcs'
+    ' --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]"'
+    ' --prefix-filecheck-ir-name _\n')
+    output_lines = NOTE + output_lines
+
+    with open(filename, 'w') as f:
+        f.write(output_lines)
+
+    # Run update_cc_test_checks.py.
+    ret = run_update_cc(filename, llvm_util_dir, llvm_bin)
+    if ret == 0:
+        # Done, return 1.
+        return 1
+    else:
+        with open(filename, 'w') as f:
+          f.write(''.join(input_lines))
+        # Failed, return 0.
+        return 0
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-j', '--jobs', type=int, default=1, help='number of parallel jobs for processing')
+    parser.add_argument('--llvm-util-dir', help='LLVM util directory with update scripts', required=True)
+    parser.add_argument('--llvm-bin', help='LLVM directory with update util scripts', required=True)
+    parser.add_argument('--debug', help='enable debug run (no multiprocessing)', action='store_true', default=False)
+    parser.add_argument('filenames', help='test filename to refactor', nargs='+')
+    args = parser.parse_args()
+
+    filenames=args.filenames
+
+    if args.debug:
+        ret = starmap(refactor, zip(filenames, repeat(args.llvm_util_dir), repeat(args.llvm_bin)))
+    else:
+        with multiprocessing.Pool(args.jobs) as pool:
+            # chunksize of 1 for maximun concurrency.
+            ret = pool.starmap(refactor, zip(filenames, repeat(args.llvm_util_dir), repeat(args.llvm_bin)), 1)
+
+    successes = sum(ret)
+
+    print('Done,', successes, '/', len(filenames), 'refactored successfully')
+
+if __name__ == "__main__":
+    main()