diff --git a/flang/examples/flang-omp-report-plugin/requirements.txt b/flang/examples/flang-omp-report-plugin/requirements.txt new file mode 100644 --- /dev/null +++ b/flang/examples/flang-omp-report-plugin/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml==0.17.16 +ruamel.yaml.clib==0.2.6 diff --git a/flang/examples/flang-omp-report-plugin/yaml_summarizer.py b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py new file mode 100644 --- /dev/null +++ b/flang/examples/flang-omp-report-plugin/yaml_summarizer.py @@ -0,0 +1,282 @@ +"""YAML Summariser + +The flang plugin ``flang-omp-report`` takes one fortran +file in and returns a YAML report file of the input file. +This becomes an issue when you want to analyse an entire project +into one final report. +The purpose of this Python script is to generate a final YAML +summary from all of the files generated by ``flang-omp-report``. + +Currently, it requires ``ruamel.yaml``, +which can be installed with: + + ``pip3 install ruamel.yaml`` + +By default it scans the directory it is ran in +for any YAML files and outputs a summary to +stdout. It can be ran as: + + ``python3 yaml_summarizer.py`` + +Parameters: + + -d --directory Specify which directory to scan. Multiple directories can be searched by + providing a semicolon seperated list of directories. + + -l --log Combine all yaml files into one log (instead of generating a summary) + + -o --output Specify a directory in which to save the summary file + + -r --recursive Recursively search directory for all yaml files + +Examples: + + ``python3 yaml_summarizer.py -d ~/llvm-project/build/ -r`` + + ``python3 yaml_summarizer.py -d "~/llvm-project/build/;~/llvm-project/flang/test/Examples"`` + + ``python3 yaml_summarizer.py -l -o ~/examples/report.yaml`` + +Pseudo-examples: + + Summary: + + $ python3 yaml_summarizer.py file_1.yaml file_2.yaml + + + Construcsts are in the form: + - construct: someOMPconstruct + count: 8 + clauses: + - clause: clauseOne + count: 4 + - clause: ClauseTwo + count: 2 + + Log: + + $ python3 yaml_summarizer.py -l file_1.yaml file_2.yaml + file_1.yaml + + file_2.yaml + + + Constructs are in the form: + - construct: someOMPConstruct + line: 12 + clauses: + - clause: clauseOne + details: 'someDetailForClause' +""" + +import sys +import glob +import argparse +from pathlib import Path +from os.path import isdir + +from ruamel.yaml import YAML + +def find_yaml_files(search_directory: Path, search_pattern: str): + """ + Find all '.yaml' files and returns an iglob iterator to them. + + Keyword arguments: + search_pattern -- Search pattern for 'iglob' to use for finding '.yaml' files. + If this is set to 'None', then it will default to just searching + for all '.yaml' files in the current directory. + """ + # @TODO: Currently *all* yaml files are read - regardless of whether they have + # been generated with 'flang-omp-report' or not. This might result in the script + # reading files that it should ignore. + if search_directory: + return glob.iglob(str(search_directory.joinpath(search_pattern)), recursive=True) + + return glob.iglob(str("/" + search_pattern), recursive=True) + +def process_log(data, result: list): + """ + Process the data input as a 'log' to the result array. This esssentially just + stitches together all of the input '.yaml' files into one result. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Array to add the processed data to. + """ + for datum in data: + items = result.get(datum['file'], []) + items.append({"construct" : datum['construct'], + "line" : datum['line'], + "clauses" : datum['clauses']}) + result[datum['file']] = items + +def add_clause(datum, construct): + """ + Add clauses to the construct if they're missing + Otherwise increment their count by one. + + Keyword arguments: + datum -- Data construct containing clauses to check. + construct -- Construct to add or increment clause count. + """ + to_check = [i['clause'] for i in construct['clauses']] + to_add = [i['clause'] for i in datum['clauses']] + clauses = construct["clauses"] + for item in to_add: + if item in to_check: + for clause in clauses: + if clause["clause"] == item: + clause["count"] += 1 + else: + clauses.append({"clause" : item, + "count" : 1}) + +def process_summary(data, result: dict): + """ + Process the data input as a 'summary' to the 'result' dictionary. + + Keyword arguments: + data -- Data from yaml.load() for a yaml file. So the type can be 'Any'. + result -- Dictionary to add the processed data to. + """ + for datum in data: + construct = next((item for item in result + if item["construct"] == datum["construct"]), None) + clauses = [] + # Add the construct and clauses to the summary if + # they haven't been seen before + if not construct: + for i in datum['clauses']: + clauses.append({"clause" : i['clause'], + "count" : 1}) + result.append({"construct" : datum['construct'], + "count" : 1, + "clauses" : clauses}) + else: + construct["count"] += 1 + + add_clause(datum, construct) + +def clean_summary(result): + """ Cleans the result after processing the yaml files with summary format.""" + # Remove all "clauses" that are empty to keep things compact + for construct in result: + if construct["clauses"] == []: + construct.pop("clauses") + +def clean_log(result): + """ Cleans the result after processing the yaml files with log format.""" + for constructs in result.values(): + for construct in constructs: + if construct["clauses"] == []: + construct.pop("clauses") + +def output_result(yaml: YAML, result, output_file: Path): + """ + Outputs result to either 'stdout' or to a output file. + + Keyword arguments: + result -- Format result to output. + output_file -- File to output result to. If this is 'None' then result will be + outputted to 'stdout'. + """ + if output_file: + with open(output_file, 'w+', encoding='utf-8') as file: + if output_file.suffix == ".yaml": + yaml.dump(result, file) + else: + file.write(result) + else: + yaml.dump(result, sys.stdout) + +def process_yaml(search_directories: list, search_pattern: str, + result_format: str, output_file: Path): + """ + Reads each yaml file, calls the appropiate format function for + the file and then ouputs the result to either 'stdout' or to an output file. + + Keyword arguments: + search_directories -- List of directory paths to search for '.yaml' files in. + search_pattern -- String pattern formatted for use with glob.iglob to find all + '.yaml' files. + result_format -- String representing output format. Current supported strings are: 'log'. + output_file -- Path to output file (If value is None, then default to outputting to 'stdout'). + """ + if result_format == "log": + result = {} + action = process_log + clean_report = clean_log + else: + result = [] + action = process_summary + clean_report = clean_summary + + yaml = YAML() + + for search_directory in search_directories: + for file in find_yaml_files(search_directory, search_pattern): + with open(file, "r", encoding='utf-8') as yaml_file: + data = yaml.load(yaml_file) + action(data, result) + + if clean_report is not None: + clean_report(result) + + output_result(yaml, result, output_file) + +def create_arg_parser(): + """ Create and return a argparse.ArgumentParser modified for script. """ + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--directory", help="Specify a directory to scan", + dest="dir", type=str) + parser.add_argument("-o", "--output", help="Writes to a file instead of\ + stdout", dest="output", type=str) + parser.add_argument("-r", "--recursive", help="Recursive search for .yaml files", + dest="recursive", type=bool, nargs='?', const=True, default=False) + + exclusive_parser = parser.add_mutually_exclusive_group() + exclusive_parser.add_argument("-l", "--log", help="Modifies report format: " + "Combines the log '.yaml' files into one file.", + action='store_true', dest='log') + return parser + +def parse_arguments(): + """ Parses arguments given to script and returns a tuple of processed arguments. """ + parser = create_arg_parser() + args = parser.parse_args() + + if args.dir: + search_directory = [Path(path) for path in args.dir.split(";")] + else: + search_directory = [Path.cwd()] + + if args.recursive: + search_pattern = "**/*.yaml" + else: + search_pattern = "*.yaml" + + if args.log: + result_format = "log" + else: + result_format = "summary" + + if args.output: + if isdir(args.output): + output_file = Path(args.output).joinpath("summary.yaml") + elif isdir(Path(args.output).resolve().parent): + output_file = Path(args.output) + else: + output_file = None + + return (search_directory, search_pattern, result_format, output_file) + +def main(): + """ Main function of script. """ + (search_directory, search_pattern, result_format, output_file) = parse_arguments() + + process_yaml(search_directory, search_pattern, result_format, output_file) + + return 0 + +if __name__ == "__main__": + sys.exit(main())