diff --git a/openmp/tools/analyzer/analyzer.py b/openmp/tools/analyzer/analyzer.py --- a/openmp/tools/analyzer/analyzer.py +++ b/openmp/tools/analyzer/analyzer.py @@ -19,7 +19,7 @@ match = re.search(r"[Uu]sed ([0-9]+) registers", usages) return match.group(1) if match else None def demangle(fn): - expr = re.compile("__omp_offloading_[a-zA-Z0-9]*_[a-zA-Z0-9]*_(_Z.*_)_l[0-9]*$") + expr = re.compile("__omp_offloading_[a-zA-Z0-9]*_[a-zA-Z0-9]*_(.*)_l[0-9]*$") match = expr.search(fn) function = match.group(1) if match else fn output = subprocess.run(demangler.split(' ') + [function], check=True, stdout=subprocess.PIPE) @@ -29,7 +29,7 @@ match = expr.search(fn) return match.group(1) if match else 0 - expr = re.compile("Function properties for \'?([a-zA-Z0-9_]*)\'?\n(.*,.*)\n") + expr = re.compile("Function properties for \'?([a-zA-Z0-9_]*)\'?\n(.*,.*)") for (fn, usages) in expr.findall(usageStr): info = usageDict[fn] if fn in usageDict else dict() info["Name"] = demangle(fn) @@ -37,7 +37,7 @@ info["Usage"] = {"Registers" : getRegisters(usages), "Shared" : getSharedMem(usages), "Kernel" : getKernelMem(usages)} usageDict[fn] = info -def getKernelUsage(stderr, fname='usage.yaml'): +def getKernelUsage(stderr, fname='kernel-usage.yaml'): remarks = [line for line in stderr.split('\n') if re.search(r"^remark:", line)] ptxas = '\n'.join([line.split(':')[1].strip() for line in stderr.split('\n') if re.search(r"^ptxas info *:", line)]) nvlink = '\n'.join([line.split(':')[1].strip() for line in stderr.split('\n') if re.search(r"^nvlink info *:", line)]) @@ -52,3 +52,9 @@ parseKernelUsages(nvlink, usage) return usage + +def runClangTidy(files, args): + # Bug in clang-tidy that crashes if -fopenmp-targets is used + args = list(filter(lambda s: not s.startswith('-fopenmp-targets'), args)) + args = list(filter(lambda s: not s.startswith('-mllvm'), args)) + output = subprocess.run(['clang-tidy'] + files + ["-checks=-*,openmp-*"] + ["--"] + args, check=True) diff --git a/openmp/tools/analyzer/llvm-openmp-analyzer b/openmp/tools/analyzer/llvm-openmp-analyzer --- a/openmp/tools/analyzer/llvm-openmp-analyzer +++ b/openmp/tools/analyzer/llvm-openmp-analyzer @@ -2,7 +2,7 @@ """ A wrapper for Clang specialized for gathering information about OpenMP programs. -Simple replace calls to clang or clang++ with llvm-openmp-analyzer to run the +Simply replace calls to clang or clang++ with llvm-openmp-analyzer to run the analysis passes. """ @@ -11,33 +11,70 @@ import yaml # PyYaml to save and load analysis information import sys import io +import re -from analyzer import getKernelUsage +import analyzer +import optrecord desc = '''A wrapper around clang that runs OpenMP Analysis passes and gathers information about OpenMP programs.''' -default_args = ["-fopenmp", "-Rpass=openmp-opt", "-Rpass-missed=openmp-opt", "-Rpass-analysis=openmp-opt"] +default_args = ['-fopenmp', '-Rpass=openmp-opt', '-Rpass-missed=openmp-opt', + '-Rpass-analysis=openmp-opt'] + +report_args = ['-fsave-optimization-record', + '-foptimization-record-passes=^(.?$|[^a].+|a[^s].+|as[^m].*)', + '-mllvm', '-openmp-print-gpu-kernels'] def main(): compiler = ["clang++"] if sys.argv[0].endswith('++') else ["clang"] parser = argparse.ArgumentParser(description=desc) parser.add_argument('--usage-report-file', metavar='filename', - default='usage.yaml', - help='Filename used for the OpenMP kernel usage reports in YAML format. "usage.yaml" by default.') + default='kernel-usage.yaml', + help='Filename used for the OpenMP kernel usage reports in YAML format. "kernel-usage.yaml" by default.') parser.add_argument('--no-usage-report', action='store_true', default=False, - help='Do not general a usage report for the OpenMP kernels.') - args, clang_args = parser.parse_known_args() + help='Do not general a usage report for the OpenMP Target kernels.') + parser.add_argument('--no-clang-tidy', + action='store_true', + default=False, + help='Do not perform clang-tidy OpenMP checks on the source files') + parser.add_argument('--no-optimization-record', + action='store_true', + default=False, + help='Do not gather optimization records') + args, forward_args = parser.parse_known_args() + + clang_args = default_args + forward_args + + subprocess.run(compiler + clang_args, check=True) + + output = subprocess.run(compiler + ['-ccc-print-phases'] + clang_args, stderr=subprocess.PIPE) + stderr = output.stderr.decode('utf-8') + files = list(set(re.findall(".*: input, \"(.*)\",.*", stderr))) + + if not args.no_optimization_record: + clang_args = clang_args + report_args - subprocess.run(compiler + default_args + clang_args, check=True) - output = subprocess.run(compiler + default_args + clang_args + ["-v"], stderr=subprocess.PIPE) + output = subprocess.run(compiler + ['-v'] + clang_args, stderr=subprocess.PIPE) stderr = output.stderr.decode('utf-8') + if not args.no_optimization_record: + opt_files = optrecord.findOptFiles(files) + remarks, file_remarks, pass_remarks, _ = optrecord.gatherResults(opt_files) + + if not args.no_clang_tidy: + analyzer.runClangTidy(files, clang_args) + if not args.no_usage_report: - usage = getKernelUsage(stderr, fname=args.usage_report_file) + usage = analyzer.getKernelUsage(stderr, fname=args.usage_report_file) + if not args.no_optimization_record: + for remark in pass_remarks['openmp-opt']['OpenMPGPU']: + if remark.Function in usage: + usage[remark.Function]['DebugLoc'] = remark.DebugLoc + with io.open(args.usage_report_file, 'w', encoding = 'utf-8') as f: yaml.dump(usage, f) diff --git a/openmp/tools/analyzer/optrecord.py b/openmp/tools/analyzer/optrecord.py new file mode 100644 --- /dev/null +++ b/openmp/tools/analyzer/optrecord.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python + +# Modified from the llvm opt-viewer tool + +from __future__ import print_function + +import io +import yaml +# Try to use the C parser. +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +import html +import fnmatch +import functools +import os, os.path +import subprocess +import re + +from collections import defaultdict +from multiprocessing import Lock +from sys import intern + +class Remark(yaml.YAMLObject): + # Work-around for http://pyyaml.org/ticket/154. + yaml_loader = Loader + + default_demangler = 'c++filt -n' + demangler_proc = None + + @classmethod + def set_demangler(cls, demangler): + cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) + cls.demangler_lock = Lock() + + @classmethod + def demangle(cls, name): + expr = re.compile("__omp_offloading_[a-zA-Z0-9]*_[a-zA-Z0-9]*_(.*)_l[0-9]*") + match = expr.search(name) + function = match.group(1) if match else name + with cls.demangler_lock: + cls.demangler_proc.stdin.write((function + '\n').encode('utf-8')) + cls.demangler_proc.stdin.flush() + return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8') + + # Intern all strings since we have lot of duplication across filenames, + # remark text. + # + # Change Args from a list of dicts to a tuple of tuples. This saves + # memory in two ways. One, a small tuple is significantly smaller than a + # small dict. Two, using tuple instead of list allows Args to be directly + # used as part of the key (in Python only immutable types are hashable). + def _reduce_memory(self): + self.Pass = intern(self.Pass) + self.Name = intern(self.Name) + try: + # Can't intern unicode strings. + self.Function = intern(self.Function) + except: + pass + + def _reduce_memory_dict(old_dict): + new_dict = dict() + for (k, v) in iter(old_dict.items()): + if type(k) is str: + k = intern(k) + + if type(v) is str: + v = intern(v) + elif type(v) is dict: + # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] + v = _reduce_memory_dict(v) + new_dict[k] = v + return tuple(new_dict.items()) + + self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) + + # The inverse operation of the dictonary-related memory optimization in + # _reduce_memory_dict. E.g. + # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] + def recover_yaml_structure(self): + def tuple_to_dict(t): + d = dict() + for (k, v) in t: + if type(v) is tuple: + v = tuple_to_dict(v) + d[k] = v + return d + + self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] + + def _GetArgString(self, mapping): + mapping = dict(list(mapping)) + (key, value) = list(mapping.items())[0] + + if key == 'Caller' or key == 'Callee' or key == 'DirectCallee': + value = self.demangle(value) + if key == 'DebugLoc': + value = "{}:{}:{}".format(value[0], value[1], value[2]) + + return value + + def canonicalize(self): + if not hasattr(self, 'Hotness'): + self.Hotness = 0 + if not hasattr(self, 'Args'): + self.Args = [] + if not hasattr(self, 'DebugLoc'): + self.DebugLoc = {'File' : 'unknown', 'Line' : 0, 'Column' : 0} + self._reduce_memory() + + @property + def File(self): + return self.DebugLoc['File'] + + @property + def Line(self): + return int(self.DebugLoc['Line']) + + @property + def Column(self): + return int(self.DebugLoc['Column']) + + @property + def DebugLocString(self): + return "{}:{}:{}".format(self.File, self.Line, self.Column) + + @property + def DemangledFunctionName(self): + return self.demangle(self.Function) + + # Return a cached dictionary for the arguments. The key for each entry is + # the argument key (e.g. 'Callee' for inlining remarks. The value is a + # list containing the value (e.g. for 'Callee' the function) and + # optionally a DebugLoc. + @property + def ArgDict(self): + if hasattr(self, '_ArgDict'): + return self._ArgDict + self._ArgDict = {} + for arg in self.Args: + if len(arg) == 2: + if arg[0][0] == 'DebugLoc': + dbgidx = 0 + else: + assert(arg[1][0] == 'DebugLoc') + dbgidx = 1 + + key = arg[1 - dbgidx][0] + entry = (arg[1 - dbgidx][1], arg[dbgidx][1]) + else: + arg = arg[0] + key = arg[0] + entry = (arg[1], ) + + self._ArgDict[key] = entry + return self._ArgDict + + @property + def Message(self): + # Args is a list of mappings (dictionaries) + values = [self._GetArgString(mapping) for mapping in self.Args] + return "".join(values) + + @property + def RelativeHotness(self): + if self.max_hotness: + return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness) + else: + return '' + + @property + def key(self): + return (self.__class__, self.Pass, self.Name, self.File, + self.Line, self.Column, self.Function, self.Args) + + def __hash__(self): + return hash(self.key) + + def __eq__(self, other): + return self.key == other.key + + def __repr__(self): + return str(self.key) + + +class Analysis(Remark): + yaml_tag = '!Analysis' + + @property + def color(self): + return "white" + + +class AnalysisFPCommute(Analysis): + yaml_tag = '!AnalysisFPCommute' + + +class AnalysisAliasing(Analysis): + yaml_tag = '!AnalysisAliasing' + + +class Passed(Remark): + yaml_tag = '!Passed' + + @property + def color(self): + return "green" + + +class Missed(Remark): + yaml_tag = '!Missed' + + @property + def color(self): + return "red" + +class Failure(Missed): + yaml_tag = '!Failure' + +def getRemarks(input_file, filter_=None): + max_hotness = 0 + all_remarks = dict() + pass_remarks = defaultdict(functools.partial(defaultdict, list)) + file_remarks = defaultdict(functools.partial(defaultdict, list)) + + filter_e = re.compile(filter_) if (filter_) else None + with io.open(input_file, encoding = 'utf-8') as f: + docs = yaml.load_all(f, Loader=yaml.Loader) + + for remark in docs: + remark.canonicalize() + # Avoid remarks withoug debug location or if they are duplicated + if not hasattr(remark, 'DebugLoc'): + continue + + if filter_e and not filter_e.search(remark.Pass): + continue + + all_remarks[remark.key] = remark; + pass_remarks[remark.Pass][remark.Name].append(remark) + file_remarks[remark.File][remark.Line].append(remark) + + if hasattr(remark, 'max_hotness'): + max_hotness = remark.max_hotness + max_hotness = max(max_hotness, remark.Hotness) + + return all_remarks, file_remarks, pass_remarks, max_hotness + +def gatherResults(files, filter_=None): + if not Remark.demangler_proc: + Remark.set_demangler(Remark.default_demangler) + + remarks = list(map(lambda file : getRemarks(file, filter_), files)) + max_hotness = max(entry[3] for entry in remarks) + + def mergeRemarks(remarks_to_merge, all_remarks, merged): + for first, d in iter(remarks_to_merge.items()): + for second, remarks in iter(d.items()): + for remark in remarks: + # Bring max_hotness into the remarks so that + # RelativeHotness does not depend on an external global. + remark.max_hotness = max_hotness + if remark.key not in all_remarks: + merged[first][second].append(remark) + + all_remarks = dict() + file_remarks = defaultdict(functools.partial(defaultdict, list)) + pass_remarks = defaultdict(functools.partial(defaultdict, list)) + for all_remarks_job, file_remarks_job, pass_remarks_job, _ in remarks: + mergeRemarks(file_remarks_job, all_remarks, file_remarks) + mergeRemarks(pass_remarks_job, all_remarks, pass_remarks) + all_remarks.update(all_remarks_job) + + return all_remarks, file_remarks, pass_remarks, max_hotness != 0 + +def findOptFiles(files): + all = [] + for filename in files: + for file in os.listdir('.'): + string = str(os.path.splitext(os.path.basename(filename))[0]) + "*.opt.yaml*" + if fnmatch.fnmatch(file, string): + all.append(file) + return all