Index: llvm/trunk/utils/opt-viewer/opt-viewer.py
===================================================================
--- llvm/trunk/utils/opt-viewer/opt-viewer.py
+++ llvm/trunk/utils/opt-viewer/opt-viewer.py
@@ -15,6 +15,13 @@
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
+
+import functools
+from collections import defaultdict
+import itertools
+from multiprocessing import Pool
+from multiprocessing import Lock, cpu_count
+import errno
import argparse
import os.path
import re
@@ -24,18 +31,14 @@
from pygments.lexers.c_cpp import CppLexer
from pygments.formatters import HtmlFormatter
-parser = argparse.ArgumentParser(description=desc)
-parser.add_argument('yaml_files', nargs='+')
-parser.add_argument('output_dir')
-parser.add_argument('-source-dir', '-s', default='', help='set source directory')
-args = parser.parse_args()
-
p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+p_lock = Lock()
def demangle(name):
- p.stdin.write(name + '\n')
- return p.stdout.readline().rstrip()
+ with p_lock:
+ p.stdin.write(name + '\n')
+ return p.stdout.readline().rstrip()
class Remark(yaml.YAMLObject):
@@ -156,16 +159,16 @@
class SourceFileRenderer:
- def __init__(self, filename):
+ def __init__(self, source_dir, output_dir, filename):
existing_filename = None
if os.path.exists(filename):
existing_filename = filename
else:
- fn = os.path.join(args.source_dir, filename)
+ fn = os.path.join(source_dir, filename)
if os.path.exists(fn):
existing_filename = fn
- self.stream = open(os.path.join(args.output_dir, SourceFileRenderer.html_file_name(filename)), 'w')
+ self.stream = open(os.path.join(output_dir, SourceFileRenderer.html_file_name(filename)), 'w')
if existing_filename:
self.source_stream = open(existing_filename)
else:
@@ -243,8 +246,8 @@
class IndexRenderer:
- def __init__(self):
- self.stream = open(os.path.join(args.output_dir, 'index.html'), 'w')
+ def __init__(self, output_dir):
+ self.stream = open(os.path.join(output_dir, 'index.html'), 'w')
def render_entry(self, r):
print('''
@@ -278,41 +281,123 @@
</html>''', file=self.stream)
-all_remarks = dict()
-file_remarks = dict()
+def get_remarks(input_file):
+ max_hotness = 0
+ all_remarks = dict()
+ file_remarks = defaultdict(functools.partial(defaultdict, list))
+
+ with open(input_file) as f:
+ docs = yaml.load_all(f, Loader=Loader)
+
+ for remark in docs:
+ # Avoid remarks withoug debug location or if they are duplicated
+ if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
+ continue
+ all_remarks[remark.key] = remark
+
+ file_remarks[remark.File][remark.Line].append(remark)
+
+ max_hotness = max(max_hotness, remark.Hotness)
+
+ return max_hotness, all_remarks, file_remarks
+
+
+def _render_file(source_dir, output_dir, entry):
+ filename, remarks = entry
+ SourceFileRenderer(source_dir, output_dir, filename).render(remarks)
+
+
+def gather_results(pool, filenames):
+ all_remarks = dict()
+ remarks = pool.map(get_remarks, filenames)
-for input_file in args.yaml_files:
- f = open(input_file)
- docs = yaml.load_all(f, Loader=Loader)
- for remark in docs:
- # Avoid remarks withoug debug location or if they are duplicated
- if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
- continue
- all_remarks[remark.key] = remark
-
- file_remarks.setdefault(remark.File, dict()).setdefault(remark.Line, []).append(remark)
-
- Remark.max_hotness = max(Remark.max_hotness, remark.Hotness)
-
-# Set up a map between function names and their source location for function where inlining happened
-for remark in all_remarks.itervalues():
- if type(remark) == Passed and remark.Pass == "inline" and remark.Name == "Inlined":
- for arg in remark.Args:
- caller = arg.get('Caller')
- if caller:
+ def merge_dicts(dicts):
+ ''' Takes an iterable of dicts and merges them into
+ a single dict. Nested dicts are merged as well.
+ >>> merge_dicts([ {'a': [3], }, {'a': [4], }, {'b': [6] }])
+ {'a': [3,4,], 'b': [6]}
+ >>> merge_dicts([ {'a': {'q': [6,3], 'f': [30],}, }, {'a': {'f': [4,10]}, }, {'b': [6] }])
+ {'a': [{'q': [6,3]}, {'f': [4,10,30]}], 'b': [6]}
+
+ '''
+ merged = defaultdict(functools.partial(defaultdict, list))
+
+ for k, v in itertools.chain(*[d.iteritems() for d in dicts]):
+ for k_, v_ in v.items():
+ merged[k][k_] += v_
+
+ return merged
+
+ file_remark_dicts = [entry[2] for entry in remarks]
+ # merge the list of remarks at each line of each file
+ file_remarks = merge_dicts(file_remark_dicts)
+
+ # merge individual 'all_remark' results:
+ for _, all_rem, _ in remarks:
+ all_remarks.update(all_rem)
+
+ Remark.max_hotness = max(entry[0] for entry in remarks)
+
+ return all_remarks, file_remarks
+
+
+def map_remarks(all_remarks):
+ # Set up a map between function names and their source location for
+ # function where inlining happened
+ for remark in all_remarks.itervalues():
+ if isinstance(remark, Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
+ for arg in remark.Args:
+ caller = arg.get('Caller')
+ if caller:
Remark.caller_loc[caller] = arg['DebugLoc']
-if Remark.should_display_hotness():
- sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True)
-else:
- sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column))
-if not os.path.exists(args.output_dir):
- os.mkdir(args.output_dir)
+def generate_report(pool, all_remarks, file_remarks, source_dir, output_dir):
+ try:
+ os.makedirs(output_dir)
+ except OSError as e:
+ if e.errno == errno.EEXIST and os.path.isdir(output_dir):
+ pass
+ else:
+ raise
+
+ _render_file_bound = functools.partial(_render_file, source_dir, output_dir)
+ pool.map(_render_file_bound, file_remarks.items())
+
+ if Remark.should_display_hotness():
+ sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: r.Hotness, reverse=True)
+ else:
+ sorted_remarks = sorted(all_remarks.itervalues(), key=lambda r: (r.File, r.Line, r.Column))
+ IndexRenderer(args.output_dir).render(sorted_remarks)
+
+ shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ "style.css"), output_dir)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description=desc)
+ parser.add_argument('yaml_files', nargs='+')
+ parser.add_argument('output_dir')
+ parser.add_argument(
+ '--jobs',
+ '-j',
+ default=cpu_count(),
+ type=int,
+ help='Max job count (defaults to current CPU count)')
+ parser.add_argument(
+ '-source-dir',
+ '-s',
+ default='',
+ help='set source directory')
+ args = parser.parse_args()
+
+ if len(args.yaml_files) == 0:
+ parser.print_help()
+ sys.exit(1)
-for (filename, remarks) in file_remarks.iteritems():
- SourceFileRenderer(filename).render(remarks)
+ pool = Pool(processes=args.jobs)
+ all_remarks, file_remarks = gather_results(pool, args.yaml_files)
-IndexRenderer().render(sorted_remarks)
+ map_remarks(all_remarks)
-shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), args.output_dir)
+ generate_report(pool, all_remarks, file_remarks, args.source_dir, args.output_dir)