diff --git a/flang/examples/flang-omp-report-plugin/yaml_summarizer.py b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py new file mode 100644 --- /dev/null +++ b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py @@ -0,0 +1,253 @@ +"""YAML Summariser + +This Python script generates a YAML summary from +the files generated by ``flang-omp-report``. +Currently, it only support ``ruamel.yaml``, +which can be installed with: + + ``pip3 install ruamel.yaml`` + +By default it scans the directory it is ran in +for any YAML files and outputs a summary to +stdout. It can be ran as: + + ``python3 yaml_summarizer.py`` + +Parameters: + + -d --directory Specify which directory to scan + + -l --log This option combines all yaml files into one + + -o --output Specify a directory in which to save the summary file + + -r --recursive Recursively search directory for all yaml files + +Examples: + + ``python3 yaml_summarizer.py -d ~/llvm-project/build/`` + + ``python3 yaml_summarizer.py -l -o ~/examples/`` +""" + +import sys +import glob +import argparse +from pathlib import Path +from os.path import isdir, isfile + +YAML_MODULE_FOUND = True +try: + from ruamel.yaml import YAML +except ImportError: + YAML_MODULE_FOUND = False + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--directory", help="Specify a directory to scan", + dest="dir", type=str) +parser.add_argument("-o", "--output", help="Writes to a file instead of\ + stdout", dest="output", type=str) +parser.add_argument("-r", "--recursive", help="Recursive search for .yaml files", + dest="recursive", type=bool, nargs='?', const=True, default=False) + +exclusive_parser = parser.add_mutually_exclusive_group() +exclusive_parser.add_argument("-l", "--log", help="Combines the log files instead of\ + writing out a summary", + action='store_true', dest='log') + +yaml = YAML() + +def find_yaml_files(search_pattern): + """ + Find all '.yaml' files and returns an iglob iterator to them. + + Keyword arguments: + search_pattern -- Search pattern for 'iglob' to use for finding '.yaml' files. + If this is set to 'None', then it will default to just searching + for all '.yaml' files in the current directory. + """ + # @TODO: Currently yaml files not from the 'flang-omp-report' plugin could be + # included resulting in errors. So we need to check whether the yaml files are + # generated in the build or source directory. If in build directory, + # look at adding some check for a prefix in yaml file name. If in source, add a + # check for the yaml file against its corresponding source file. + if search_pattern: + return glob.iglob(search_pattern, recursive=True) + + return glob.iglob('*.yaml') + +def process_log(data, result): + """ + Process the data input as a 'log' to the result array. This esssentially just + stitches together all of the '.yaml' input files into one result. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Array to add the processed data to. + + Example after being processed to yaml: + /../llvm-project/flang/test/Examples/omp-device-constructs.f90: + - construct: target + line: 18 + clauses: + - clause: map + details: arraya + - construct: target + line: 24 + clauses: + - clause: device + details: '0' + """ + for datum in data: + items = result.get(datum['file'], []) + items.append({"construct" : datum['construct'], + "line" : datum['line'], + "clauses" : datum['clauses']}) + result[datum['file']] = items + +def add_clause(datum, construct): + """ + Add clauses to the construct if they're missing + Otherwise increment their count by one. + + Keyword arguments: + dataum -- Data construct containing clauses to check. + construct -- Construct to add or increment clause count. + """ + to_check = [i['clause'] for i in construct['clauses']] + to_add = [i['clause'] for i in datum['clauses']] + clauses = construct["clauses"] + for item in to_add: + if item in to_check: + for clause in clauses: + if clause["clause"] == item: + clause["num"] += 1 + else: + clauses.append({"clause" : item, + "num" : 1}) + +def process_summary(data, result): + """ + Process the data input as a 'summary' to the 'result' dictionary. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Dictionary to add the processed data to. + + Example after being processed to yaml: + - construct: target data + num: 1 + clauses: + - clause: device + num: 1 + - clause: map + num: 1 + - construct: atomic-read + num: 1 + clauses: + - clause: seq_cst + num: 1 + - construct: atomic-write + num: 1 + clauses: + - clause: seq_cst + num: 1 + """ + for datum in data: + construct = next((item for item in result + if item["construct"] == datum["construct"]), None) + clauses = [] + # Add the construct and clauses to the summary if + # they haven't been seen before + if not construct: + for i in datum['clauses']: + clauses.append({"clause" : i['clause'], + "num" : 1}) + result.append({"construct" : datum['construct'], + "num" : 1, + "clauses" : clauses}) + else: + construct["num"] += 1 + + add_clause(datum, construct) + +def output_result(result, output_file=None): + """ + Outputs result to either 'stdout' or to a output file. + + Keyword arguments: + result -- Format result to output. + output_file -- File to output result to. If this is 'None' then result will be + outputted to 'stdout'. (Default: None) + """ + if output_file: + with open(output_file, 'w+', encoding='utf-8') as file: + if output_file.endswith(".yaml"): + yaml.dump(result, file) + else: + file.write(result) + else: + yaml.dump(result, sys.stdout) + + +def process_yaml(search_pattern=None, result_format=None, output_file=None): + """ + Reads each yaml file, calls the appropiate format function for + the file and then ouputs the result to either 'stdout' or to an output file. + + Keyword arguments: + search_pattern -- String pattern formatted for use with glob.iglob to find all + '.yaml' files. (Default: None) + result_format -- String representing output format. Current supported strings are: 'log'. + (Default: None - The format will default to 'summary' when argument is None) + output_file -- Path to output file. (Default: None) + (If value is None, then defaults to outputting to 'stdout') + """ + if result_format == "log": + result = {} + action = process_log + else: + result = [] + action = process_summary + + for file in find_yaml_files(search_pattern): + with open(file, "r", encoding='utf-8') as yaml_file: + data = yaml.load(yaml_file) + action(data, result) + + output_result(result, output_file) + +def yaml_module_not_found(): + """ Print missing yaml module error and exit script. """ + print("Currently this script only works with\ + ``ruamel.yaml`` installed.") + sys.exit(1) + +if __name__ == "__main__": + if not YAML_MODULE_FOUND: + yaml_module_not_found() + + args = parser.parse_args() + + SEARCH_PATTERN = None + if args.dir: + if args.recursive: + SEARCH_PATTERN = str(Path(args.dir).joinpath("**/*.yaml")) + else: + SEARCH_PATTERN = str(Path(args.dir).joinpath("*.yaml")) + + RESULT_FORMAT = None + if args.log: + RESULT_FORMAT = "log" + else: + RESULT_FORMAT = "summary" + + OUTPUT_FILE = None + if args.output: + if isdir(args.output): + OUTPUT_FILE = Path(args.output).joinpath("summary.yaml") + elif isfile(args.output): + OUTPUT_FILE = Path(args.output) + + + process_yaml(SEARCH_PATTERN, RESULT_FORMAT, OUTPUT_FILE)