Index: utils/opt-viewer/opt-viewer.py =================================================================== --- utils/opt-viewer/opt-viewer.py +++ utils/opt-viewer/opt-viewer.py @@ -13,9 +13,14 @@ # Try to use the C parser. try: from yaml import CLoader as Loader + # no GIL contention means multithreading should work + from multiprocessing.pool import ThreadPool as Pool except ImportError: from yaml import Loader + from multiprocessing import Pool import argparse +import os +import errno import os.path import re import subprocess @@ -23,6 +28,7 @@ from pygments import highlight from pygments.lexers.c_cpp import CppLexer from pygments.formatters import HtmlFormatter +import time parser = argparse.ArgumentParser(description=desc) parser.add_argument('yaml_files', nargs='+') @@ -176,7 +182,7 @@ '''.format(filename), file=self.stream) - self.html_formatter = HtmlFormatter() + self.html_formatter = HtmlFormatter(encoding='utf-8') self.cpp_lexer = CppLexer() def render_source_line(self, linenum, line): @@ -278,41 +284,85 @@ ''', file=self.stream) -all_remarks = dict() -file_remarks = dict() -for input_file in args.yaml_files: - f = open(input_file) - docs = yaml.load_all(f, Loader=Loader) - for remark in docs: - # Avoid remarks withoug debug location or if they are duplicated - if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: - continue - all_remarks[remark.key] = remark +import functools +from collections import defaultdict +def get_remarks(input_file): + max_hotness = 0 + all_remarks = dict() + file_remarks = defaultdict(functools.partial(defaultdict, list)) + + with open(input_file) as f: + docs = yaml.load_all(f, Loader=Loader) - file_remarks.setdefault(remark.File, dict()).setdefault(remark.Line, []).append(remark) + for remark in docs: + # Avoid remarks withoug debug location or if they are duplicated + if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: + continue + all_remarks[remark.key] = remark - Remark.max_hotness = max(Remark.max_hotness, remark.Hotness) + file_remarks[remark.File][remark.Line].append(remark) -# Set up a map between function names and their source location for function where inlining happened -for remark in all_remarks.itervalues(): - if type(remark) == Passed and remark.Pass == "inline" and remark.Name == "Inlined": - for arg in remark.Args: - caller = arg.get('Caller') - if caller: - Remark.caller_loc[caller] = arg['DebugLoc'] + max_hotness = max(max_hotness, remark.Hotness) -if Remark.should_display_hotness(): - sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True) -else: - sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column)) + return max_hotness, all_remarks, file_remarks -if not os.path.exists(args.output_dir): - os.mkdir(args.output_dir) -for (filename, remarks) in file_remarks.iteritems(): +def render_file(entry): + filename, remarks = entry SourceFileRenderer(filename).render(remarks) -IndexRenderer().render(sorted_remarks) +if __name__ == '__main__': + t_begin = time.time() + + t0 = time.time() + pool = Pool(processes=20) + all_remarks = dict() + file_remarks = defaultdict(functools.partial(defaultdict, list)) + remarks = pool.map(get_remarks, args.yaml_files) + + # merge results: + for hotness, all_rem, file_rem in remarks: + all_remarks.update(all_rem) + file_remarks.update(file_rem) + Remark.max_hotness = max(Remark.max_hotness, hotness) + + t_end = time.time() + print('processed input files', t_end-t0, 'secs') + + # Set up a map between function names and their source location for function where inlining happened + for remark in all_remarks.itervalues(): + if type(remark) == Passed and remark.Pass == "inline" and remark.Name == "Inlined": + for arg in remark.Args: + caller = arg.get('Caller') + if caller: + Remark.caller_loc[caller] = arg['DebugLoc'] + t_end_map = time.time() + print('mapped remarks', t_end_map-t_end, 'secs') + + if Remark.should_display_hotness(): + sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True) + else: + sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column)) + + try: + os.makedirs(args.output_dir) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(args.output_dir): + pass + else: + raise + + t0 = time.time() + pool.map(render_file, file_remarks.items()) + t_end = time.time() + + print('file rendering', t_end-t0, 'sec') + IndexRenderer().render(sorted_remarks) + t_index_end = time.time() + + print('index rendering', t_index_end - t_end, 'sec') + shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), args.output_dir) -shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), args.output_dir) + t_ultimate_end = time.time() + print('total time', t_ultimate_end - t_begin, 'secs')