diff --git a/flang/examples/flang-omp-report-plugin/requirements.txt b/flang/examples/flang-omp-report-plugin/requirements.txt new file mode 100644 --- /dev/null +++ b/flang/examples/flang-omp-report-plugin/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml==0.17.16 +ruamel.yaml.clib==0.2.6 diff --git a/flang/examples/flang-omp-report-plugin/yaml_summarizer.py b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py new file mode 100644 --- /dev/null +++ b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py @@ -0,0 +1,255 @@ +"""YAML Summariser + +This Python script generates a YAML summary from +the files generated by ``flang-omp-report``. +Currently, it only support ``ruamel.yaml``, +which can be installed with: + + ``pip3 install ruamel.yaml`` + +By default it scans the directory it is ran in +for any YAML files and outputs a summary to +stdout. It can be ran as: + + ``python3 yaml_summarizer.py`` + +Parameters: + + -d --directory Specify which directory to scan + + -l --log This option combines all yaml files into one + + -o --output Specify a directory in which to save the summary file + + -r --recursive Recursively search directory for all yaml files + +Examples: + + ``python3 yaml_summarizer.py -d ~/llvm-project/build/`` + + ``python3 yaml_summarizer.py -l -o ~/examples/`` +""" + +import sys +import glob +import argparse +from pathlib import Path +from os.path import isdir, isfile + +from ruamel.yaml import YAML + +def find_yaml_files(search_pattern): + """ + Find all '.yaml' files and returns an iglob iterator to them. + + Keyword arguments: + search_pattern -- Search pattern for 'iglob' to use for finding '.yaml' files. + If this is set to 'None', then it will default to just searching + for all '.yaml' files in the current directory. + """ + # @TODO: Currently yaml files not from the 'flang-omp-report' plugin could be + # included resulting in errors. So we need to check whether the yaml files are + # generated in the build or source directory. If in build directory, + # look at adding some check for a prefix in yaml file name. If in source, add a + # check for the yaml file against its corresponding source file. + if search_pattern: + return glob.iglob(search_pattern, recursive=True) + + return glob.iglob('*.yaml') + +def process_log(data, result): + """ + Process the data input as a 'log' to the result array. This esssentially just + stitches together all of the '.yaml' input files into one result. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Array to add the processed data to. + + Example after being processed to yaml: + /../llvm-project/flang/test/Examples/omp-device-constructs.f90: + - construct: target + line: 18 + clauses: + - clause: map + details: arraya + - construct: target + line: 24 + clauses: + - clause: device + details: '0' + """ + for datum in data: + items = result.get(datum['file'], []) + items.append({"construct" : datum['construct'], + "line" : datum['line'], + "clauses" : datum['clauses']}) + result[datum['file']] = items + +def add_clause(datum, construct): + """ + Add clauses to the construct if they're missing + Otherwise increment their count by one. + + Keyword arguments: + dataum -- Data construct containing clauses to check. + construct -- Construct to add or increment clause count. + """ + to_check = [i['clause'] for i in construct['clauses']] + to_add = [i['clause'] for i in datum['clauses']] + clauses = construct["clauses"] + for item in to_add: + if item in to_check: + for clause in clauses: + if clause["clause"] == item: + clause["num"] += 1 + else: + clauses.append({"clause" : item, + "num" : 1}) + +def process_summary(data, result): + """ + Process the data input as a 'summary' to the 'result' dictionary. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Dictionary to add the processed data to. + + Example after being processed to yaml: + - construct: target data + num: 1 + clauses: + - clause: device + num: 1 + - clause: map + num: 1 + - construct: atomic-read + num: 1 + clauses: + - clause: seq_cst + num: 1 + - construct: atomic-write + num: 1 + clauses: + - clause: seq_cst + num: 1 + """ + for datum in data: + construct = next((item for item in result + if item["construct"] == datum["construct"]), None) + clauses = [] + # Add the construct and clauses to the summary if + # they haven't been seen before + if not construct: + for i in datum['clauses']: + clauses.append({"clause" : i['clause'], + "num" : 1}) + result.append({"construct" : datum['construct'], + "num" : 1, + "clauses" : clauses}) + else: + construct["num"] += 1 + + add_clause(datum, construct) + +def output_result(yaml, result, output_file=None): + """ + Outputs result to either 'stdout' or to a output file. + + Keyword arguments: + result -- Format result to output. + output_file -- File to output result to. If this is 'None' then result will be + outputted to 'stdout'. (Default: None) + """ + if output_file: + with open(output_file, 'w+', encoding='utf-8') as file: + if output_file.endswith(".yaml"): + yaml.dump(result, file) + else: + file.write(result) + else: + yaml.dump(result, sys.stdout) + + +def process_yaml(search_pattern=None, result_format=None, output_file=None): + """ + Reads each yaml file, calls the appropiate format function for + the file and then ouputs the result to either 'stdout' or to an output file. + + Keyword arguments: + search_pattern -- String pattern formatted for use with glob.iglob to find all + '.yaml' files. (Default: None) + result_format -- String representing output format. Current supported strings are: 'log'. + (Default: None - The format will default to 'summary' when argument is None) + output_file -- Path to output file. (Default: None) + (If value is None, then defaults to outputting to 'stdout') + """ + if result_format == "log": + result = {} + action = process_log + else: + result = [] + action = process_summary + + yaml = YAML() + + for file in find_yaml_files(search_pattern): + with open(file, "r", encoding='utf-8') as yaml_file: + data = yaml.load(yaml_file) + action(data, result) + + output_result(yaml, result, output_file) + +def create_arg_parser(): + """ Create and return a argparse.ArgumentParser modified for script. """ + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--directory", help="Specify a directory to scan", + dest="dir", type=str) + parser.add_argument("-o", "--output", help="Writes to a file instead of\ + stdout", dest="output", type=str) + parser.add_argument("-r", "--recursive", help="Recursive search for .yaml files", + dest="recursive", type=bool, nargs='?', const=True, default=False) + + exclusive_parser = parser.add_mutually_exclusive_group() + exclusive_parser.add_argument("-l", "--log", help="Modifies report format: " + "Combines the log '.yaml' files into one file.", + action='store_true', dest='log') + return parser + +def parse_arguments(): + """ Parses arguments given to script and returns a tuple of processed arguments. """ + parser = create_arg_parser() + args = parser.parse_args() + + search_pattern = None + if args.dir: + if args.recursive: + search_pattern = str(Path(args.dir).joinpath("**/*.yaml")) + else: + search_pattern = str(Path(args.dir).joinpath("*.yaml")) + + result_format = None + if args.log: + result_format = "log" + else: + result_format = "summary" + + output_file = None + if args.output: + if isdir(args.output): + output_file = Path(args.output).joinpath("summary.yaml") + elif isfile(args.output): + output_file = Path(args.output) + + return (search_pattern, result_format, output_file) + +def main(): + """ Main function of script. """ + (search_pattern, result_format, output_file) = parse_arguments() + + process_yaml(search_pattern, result_format, output_file) + + return 0 + +if __name__ == "__main__": + sys.exit(main())