Index: utils/opt-viewer/opt-viewer.py =================================================================== --- utils/opt-viewer/opt-viewer.py +++ utils/opt-viewer/opt-viewer.py @@ -15,7 +15,13 @@ from yaml import CLoader as Loader except ImportError: from yaml import Loader + +from multiprocessing import Pool +from multiprocessing import Lock, cpu_count + import argparse +import errno +import os import os.path import re import subprocess @@ -23,19 +29,20 @@ from pygments import highlight from pygments.lexers.c_cpp import CppLexer from pygments.formatters import HtmlFormatter +import functools +from collections import defaultdict +import itertools -parser = argparse.ArgumentParser(description=desc) -parser.add_argument('yaml_files', nargs='+') -parser.add_argument('output_dir') -parser.add_argument('-source-dir', '-s', default='', help='set source directory') -args = parser.parse_args() -p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) +p = subprocess.Popen( + ['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) +p_lock = Lock() def demangle(name): - p.stdin.write(name + '\n') - return p.stdout.readline().rstrip() + with p_lock: + p.stdin.write(name + '\n') + return p.stdout.readline().rstrip() class Remark(yaml.YAMLObject): @@ -50,14 +57,16 @@ # missing and no relative hotness information is displayed return cls.max_hotness != 0 - # Map function names to their source location for function where inlining happened + # Map function names to their source location for function where inlining + # happened caller_loc = dict() def __getattr__(self, name): # If hotness is missing, assume 0 if name == 'Hotness': return 0 - raise AttributeError + + raise AttributeError("'Remark' object has no attribute: '{}'".format(name)) @property def File(self): @@ -112,15 +121,26 @@ return "".join(values) @property + def relative_hotness_pct(self): + return int(round(self.Hotness * 100 / Remark.max_hotness)) + + @property def RelativeHotness(self): if Remark.should_display_hotness(): - return "{}%".format(int(round(self.Hotness * 100 / Remark.max_hotness))) + return "{}%".format(self.relative_hotness_pct) else: return '' @property def key(self): - return (self.__class__, self.Pass, self.Name, self.File, self.Line, self.Column, self.message) + return ( + self.__class__, + self.Pass, + self.Name, + self.File, + self.Line, + self.Column, + self.message) class Analysis(Remark): @@ -156,16 +176,20 @@ class SourceFileRenderer: - def __init__(self, filename): + def __init__(self, source_dir, output_dir, filename): existing_filename = None if os.path.exists(filename): existing_filename = filename else: - fn = os.path.join(args.source_dir, filename) + fn = os.path.join(source_dir, filename) if os.path.exists(fn): existing_filename = fn - self.stream = open(os.path.join(args.output_dir, SourceFileRenderer.html_file_name(filename)), 'w') + self.stream = open( + os.path.join( + output_dir, + SourceFileRenderer.html_file_name(filename)), + 'w') if existing_filename: self.source_stream = open(existing_filename) else: @@ -176,7 +200,7 @@ '''.format(filename), file=self.stream) - self.html_formatter = HtmlFormatter() + self.html_formatter = HtmlFormatter(encoding='utf-8') self.cpp_lexer = CppLexer() def render_source_line(self, linenum, line): @@ -194,7 +218,8 @@ dl = Remark.caller_loc.get(r.Function) if dl: link = Remark.make_link(dl['File'], dl['Line'] - 2) - inlining_context = "{r.DemangledFunctionName}".format(**locals()) + inlining_context = "{r.DemangledFunctionName}".format( + **locals()) # Column is the number of characters *including* tabs, keep those and # replace everything else with spaces. @@ -228,7 +253,11 @@ Source Inline Context ''', file=self.stream) - for (linenum, line) in enumerate(self.source_stream.readlines(), start=1): + for ( + linenum, + line) in enumerate( + self.source_stream.readlines(), + start=1): self.render_source_line(linenum, line) for remark in line_remarks.get(linenum, []): self.render_inline_remarks(remark, line) @@ -243,8 +272,9 @@ class IndexRenderer: - def __init__(self): - self.stream = open(os.path.join(args.output_dir, 'index.html'), 'w') + + def __init__(self, output_dir): + self.stream = open(os.path.join(output_dir, 'index.html'), 'w') def render_entry(self, r): print(''' @@ -278,41 +308,120 @@ ''', file=self.stream) -all_remarks = dict() -file_remarks = dict() +def get_remarks(input_file): + max_hotness = 0 + all_remarks = dict() + file_remarks = defaultdict(functools.partial(defaultdict, list)) + + with open(input_file) as f: + docs = yaml.load_all(f, Loader=Loader) + + for remark in docs: + # Avoid remarks withoug debug location or if they are duplicated + if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: + continue + all_remarks[remark.key] = remark + + file_remarks[remark.File][remark.Line].append(remark) + + max_hotness = max(max_hotness, remark.Hotness) + + return max_hotness, all_remarks, file_remarks + + +def _render_file(source_dir, output_dir, entry): + try: + filename, remarks = entry + SourceFileRenderer(source_dir, output_dir, filename).render(remarks) + except Exception as e: + print('detail', e) + -for input_file in args.yaml_files: - f = open(input_file) - docs = yaml.load_all(f, Loader=Loader) - for remark in docs: - # Avoid remarks withoug debug location or if they are duplicated - if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: - continue - all_remarks[remark.key] = remark +def gather_results(filenames): + pool = Pool(processes=cpu_count()) + all_remarks = dict() + file_remarks = defaultdict(functools.partial(defaultdict, list)) + remarks = pool.map(get_remarks, filenames) - file_remarks.setdefault(remark.File, dict()).setdefault(remark.Line, []).append(remark) + def merge_dicts(dicts): + merged = defaultdict(functools.partial(defaultdict, list)) - Remark.max_hotness = max(Remark.max_hotness, remark.Hotness) + for k, v in itertools.chain(*[d.iteritems() for d in dicts]): + for k_, v_ in v.items(): + merged[k][k_] += v_ -# Set up a map between function names and their source location for function where inlining happened -for remark in all_remarks.itervalues(): - if type(remark) == Passed and remark.Pass == "inline" and remark.Name == "Inlined": - for arg in remark.Args: - caller = arg.get('Caller') - if caller: + return merged + + file_remark_dicts = [entry[2] for entry in remarks] + file_remarks = merge_dicts(file_remark_dicts) + + # merge results: + for hotness, all_rem, file_rem in remarks: + all_remarks.update(all_rem) + + Remark.max_hotness = max(entry[0] for entry in remarks) + + return all_remarks, file_remarks + + +def map_remarks(all_remarks): + # Set up a map between function names and their source location for + # function where inlining happened + for remark in all_remarks.itervalues(): + if isinstance(remark, Passed) and remark.Pass == "inline" and remark.Name == "Inlined": + for arg in remark.Args: + caller = arg.get('Caller') + if caller: Remark.caller_loc[caller] = arg['DebugLoc'] -if Remark.should_display_hotness(): - sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True) -else: - sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column)) -if not os.path.exists(args.output_dir): - os.mkdir(args.output_dir) +def generate_report(all_remarks, output_dir): + if Remark.should_display_hotness(): + sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, + reverse=True) + else: + sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: ( + r.File, r.Line, r.Column)) + + try: + os.makedirs(output_dir) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(output_dir): + pass + else: + raise + + pool = Pool(processes=cpu_count()) + _render_file_bound = functools.partial(_render_file, args.source_dir, args.output_dir) + pool.map(_render_file_bound, file_remarks.items()) + + IndexRenderer(args.output_dir).render(sorted_remarks) + + shutil.copy( + os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + "style.css"), + output_dir) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('yaml_files', nargs='+') + parser.add_argument('output_dir') + parser.add_argument( + '-source-dir', + '-s', + default='', + help='set source directory') + args = parser.parse_args() + + if len(args.yaml_files) == 0: + parser.print_help() + sys.exit(1) -for (filename, remarks) in file_remarks.iteritems(): - SourceFileRenderer(filename).render(remarks) + all_remarks, file_remarks = gather_results(args.yaml_files) -IndexRenderer().render(sorted_remarks) + map_remarks(all_remarks) -shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), args.output_dir) + generate_report(all_remarks, args.output_dir)