'),o.g=[],o.c=[],o.w=o.width(),o.gc=o.prev(),o.f=o.opt.fixed,i.marginLeft&&o.gc.css("marginLeft",i.marginLeft),i.marginRight&&o.gc.css("marginRight",i.marginRight),o.cs=f(p?e.cellSpacing||e.currentStyle.borderSpacing:o.css("border-spacing"))||2,o.b=f(p?e.border||e.currentStyle.borderLeftWidth:o.css("border-left-width"))||1,s[a]=o,v(o))},w=function(t){var e=t.attr(n),t=s[e];t&&t.is("table")&&(t.removeClass(l+" "+c).gc.remove(),delete s[e])},v=function(i){var r=i.find(">thead>tr:first>th,>thead>tr:first>td");r.length||(r=i.find(">tbody>tr:first>th,>tr:first>th,>tbody>tr:first>td, >tr:first>td")),r=r.filter(":visible"),i.cg=i.find("col"),i.ln=r.length,i.p&&e&&e[i.id]&&m(i,r),r.each(function(e){var r=t(this),o=-1!=i.dc.indexOf(e),s=t(i.gc.append('

')[0].lastChild);s.append(o?"":i.opt.gripInnerHtml).append('

'),e==i.ln-1&&(s.addClass("JCLRLastGrip"),i.f&&s.html("")),s.bind("touchstart mousedown",J),o?s.addClass("JCLRdisabledGrip"):s.removeClass("JCLRdisabledGrip").bind("touchstart mousedown",J),s.t=i,s.i=e,s.c=r,r.w=r.width(),i.g.push(s),i.c.push(r),r.width(r.w).removeAttr("width"),s.data(l,{i:e,t:i.attr(n),last:e==i.ln-1})}),i.cg.removeAttr("width"),i.find("td, th").not(r).not("table th, table td").each(function(){t(this).removeAttr("width")}),i.f||i.removeAttr("width").addClass(c),C(i)},m=function(t,i){var r,o,s=0,d=0,n=[];if(i){if(t.cg.removeAttr("width"),t.opt.flush)return void(e[t.id]="");for(r=e[t.id].split(";"),o=r[t.ln+1],!t.f&&o&&(t.width(o*=1),t.opt.overflow&&(t.css("min-width",o+a),t.w=o));d*{cursor:"+n.opt.dragCursor+"!important}"),a.addClass(n.opt.draggingClass),o=a,n.c[d.i].l)for(var f,h=0;h> style.css */ + +.hll { background-color: #ffffcc } +.c { color: #888888 } /* Comment */ +.err { color: #FF0000; background-color: #FFAAAA } /* Error */ +.k { color: #008800; font-weight: bold } /* Keyword */ +.o { color: #333333 } /* Operator */ +.ch { color: #888888 } /* Comment.Hashbang */ +.cm { color: #888888 } /* Comment.Multiline */ +.cp { color: #557799 } /* Comment.Preproc */ +.cpf { color: #888888 } /* Comment.PreprocFile */ +.c1 { color: #888888 } /* Comment.Single */ +.cs { color: #cc0000; font-weight: bold } /* Comment.Special */ +.gd { color: #A00000 } /* Generic.Deleted */ +.ge { font-style: italic } /* Generic.Emph */ +.gr { color: #FF0000 } /* Generic.Error */ +.gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.gi { color: #00A000 } /* Generic.Inserted */ +.go { color: #888888 } /* Generic.Output */ +.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ +.gs { font-weight: bold } /* Generic.Strong */ +.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.gt { color: #0044DD } /* Generic.Traceback */ +.kc { color: #008800; font-weight: bold } /* Keyword.Constant */ +.kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ +.kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ +.kp { color: #003388; font-weight: bold } /* Keyword.Pseudo */ +.kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ +.kt { color: #333399; font-weight: bold } /* Keyword.Type */ +.m { color: #6600EE; font-weight: bold } /* Literal.Number */ +.s { background-color: #fff0f0 } /* Literal.String */ +.na { color: #0000CC } /* Name.Attribute */ +.nb { color: #007020 } /* Name.Builtin */ +.nc { color: #BB0066; font-weight: bold } /* Name.Class */ +.no { color: #003366; font-weight: bold } /* Name.Constant */ +.nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.ni { color: #880000; font-weight: bold } /* Name.Entity */ +.ne { color: #FF0000; font-weight: bold } /* Name.Exception */ +.nf { color: #0066BB; font-weight: bold } /* Name.Function */ +.nl { color: #997700; font-weight: bold } /* Name.Label */ +.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.nt { color: #007700 } /* Name.Tag */ +.nv { color: #996633 } /* Name.Variable */ +.ow { color: #000000; font-weight: bold } /* Operator.Word */ +.w { color: #bbbbbb } /* Text.Whitespace */ +.mb { color: #6600EE; font-weight: bold } /* Literal.Number.Bin */ +.mf { color: #6600EE; font-weight: bold } /* Literal.Number.Float */ +.mh { color: #005588; font-weight: bold } /* Literal.Number.Hex */ +.mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ +.mo { color: #4400EE; font-weight: bold } /* Literal.Number.Oct */ +.sb { background-color: #fff0f0 } /* Literal.String.Backtick */ +.sc { color: #0044DD } /* Literal.String.Char */ +.sd { color: #DD4422 } /* Literal.String.Doc */ +.s2 { background-color: #fff0f0 } /* Literal.String.Double */ +.se { color: #666666; font-weight: bold; background-color: #fff0f0 } /* Literal.String.Escape */ +.sh { background-color: #fff0f0 } /* Literal.String.Heredoc */ +.si { background-color: #eeeeee } /* Literal.String.Interpol */ +.sx { color: #DD2200; background-color: #fff0f0 } /* Literal.String.Other */ +.sr { color: #000000; background-color: #fff0ff } /* Literal.String.Regex */ +.s1 { background-color: #fff0f0 } /* Literal.String.Single */ +.ss { color: #AA6600 } /* Literal.String.Symbol */ +.bp { color: #007020 } /* Name.Builtin.Pseudo */ +.vc { color: #336699 } /* Name.Variable.Class */ +.vg { color: #dd7700; font-weight: bold } /* Name.Variable.Global */ +.vi { color: #3333BB } /* Name.Variable.Instance */ +.il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ + +table#opt_table.dataTable td { + font-family: monospace; +} + +table#opt_table_code.dataTable td { + padding: 0px; + border: 0px; +} + +.back { + background-color: #04AA6D; + color: white; + text-decoration: none; + display: inline-block; + padding: 8px 16px; +} + +h1.filename-title { + text-align:center; +} + +.indent-span { + -webkit-box-decoration-break: clone; + box-decoration-break: clone; + font-family: monospace; +} + +table.dataTable thead th[data-is-resizable=true] { + border-left: 1px solid transparent; + border-right: 1px dashed #bfbfbf; +} + +table.dataTable thead th.dt-colresizable-hover { + cursor: col-resize; + background-color: #eaeaea; + border-left: 1px solid #bfbfbf; + } + +table.dataTable thead th.dt-colresizable-bound-min, +table.dataTable thead th.dt-colresizable-bound-max { + opacity: 0.2; + cursor: not-allowed !important; +} \ No newline at end of file Index: llvm/tools/opt-viewer/config.yaml =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/config.yaml @@ -0,0 +1,21 @@ +--- +## Opt view config + +# Set this to false to disable remark_filters list +use_remark_filters: true + +# Exclude optimization remarks with names matching this regex. +# Empirical list of remark names (as of clang14): +# LoopSpillReloadCopies, LoadWithLoopInvariantAddressInvalidated, MissedDetails, LoadClobbered, +# NotBeneficial, SpillReloadCopies, NotPossible, NoDefinition, NeverInline, UnsupportedIrreducibleCFG, +# VectorizationNotBeneficial, LoadWithLoopInvariantAddressCondExecuted, HorSLPNotBeneficial, TooCostly +# LoopMayAccessStore +exclude_names: NeverInline|NotPossible|LoopSpillReloadCopies|SpillReloadCopies +# Exclude optimization remarks with text matching this regex: +exclude_text: ^std\:\:|^__dynamic_cast|^operator new|^operator delete|^__cxa|^__clang|^__cxx + +# Collect all optimization remarks, not just failures +collect_opt_success: False + +# Annotate all files, including system headers +annotate_external: false Index: llvm/tools/opt-viewer/config_parser.py =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/config_parser.py @@ -0,0 +1,16 @@ +import yaml +import re + +def parse(f): + "Parse config file" + # TODO - add scheme; currently we just take whatever is there. + config = yaml.safe_load(f) + if config.get('use_remark_filters', True) and 'remark_filters' in config and len(config['remark_filters']) > 0: + # config wants a single regex, transform to a single expression + try: + regexes = [re.compile(x) for x in config['remark_filters']] + except Exception as ex: + raise Exception(f"Failed to parse regex in remarks_filters config. Details {ex}") + config['remark_filter'] = '|'.join(x.pattern for x in regexes) + + return config Index: llvm/tools/opt-viewer/cpp_optimization_example/.gitignore =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/cpp_optimization_example/.gitignore @@ -0,0 +1,6 @@ +example +*.o +yaml_optimization_remarks +.depend +.vscode +html_output Index: llvm/tools/opt-viewer/cpp_optimization_example/Makefile =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/cpp_optimization_example/Makefile @@ -0,0 +1,43 @@ +CC=clang++ +CXX=clang++ +RM=rm -f +MKDIR_P=mkdir -p +OBJS_DIR=objs +DEPENDS_FILE=${OBJS_DIR}/.depend +YAML_OUTPUT_DIR=yaml_optimization_remarks +OPTIMIZATION_RECORD_FLAGS=-fsave-optimization-record -foptimization-record-file=./${YAML_OUTPUT_DIR}/$(patsubst %.o,%.opt.yaml,$(notdir $@)) +CPPFLAGS=-O3 -std=c++17 +LDFLAGS= +LDLIBS= +BINARY_NAME=example + +SRCS=$(wildcard *.cc) +OBJS=$(addprefix $(OBJS_DIR)/,$(subst .cc,.o,$(SRCS))) + +.PHONY: output_folder + +all: output_folder ${BINARY_NAME} + +directories: ${YAML_OUTPUT_DIR} ${OBJS_DIR} + +${YAML_OUTPUT_DIR} ${OBJS_DIR}: + ${MKDIR_P} $@ + +${BINARY_NAME}: $(OBJS) + $(CXX) $(LDFLAGS) -o $(BINARY_NAME) $(OBJS) $(LDLIBS) + +${OBJS_DIR}/%.o : %.cc + $(CXX) -c $(CFLAGS) $(CPPFLAGS) $(OPTIMIZATION_RECORD_FLAGS) $< -o $@ + +depend: $(DEPENDS_FILE) + +$(DEPENDS_FILE): $(SRCS) + $(RM) $(DEPENDS_FILE) + $(MKDIR_P) ${YAML_OUTPUT_DIR} ${OBJS_DIR} + $(CXX) $(CPPFLAGS) -MM $^ >> $(DEPENDS_FILE) + +include $(DEPENDS_FILE) + +clean: + $(RM) -r ${YAML_OUTPUT_DIR} ${OBJS_DIR} + Index: llvm/tools/opt-viewer/cpp_optimization_example/main.cc =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/cpp_optimization_example/main.cc @@ -0,0 +1,21 @@ +#include +#include + +// Based on Roi Barkan - "Argument passing, core guidelines and concepts" - https://www.youtube.com/watch?v=uylFACqcWYI +void scale_down(std::vector& v, const double& a) { + for (auto& item : v) { + item /= a; + } +} + +void scale_down_example() { + std::vector v {2, 1, 2, 3, 4}; + scale_down(v, v[0]); + +} + +int main() { + scale_down_example(); + + return 0; +} Index: llvm/tools/opt-viewer/cpp_optimization_example/run_optview2.sh =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/cpp_optimization_example/run_optview2.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" || exit 1 + +echo "Running make..." +make + +echo "Running optview2..." +../opt-viewer.py --open-browser --output-dir ./html_output --source-dir ./ ./yaml_optimization_remarks Index: llvm/tools/opt-viewer/opt-viewer.py =================================================================== --- llvm/tools/opt-viewer/opt-viewer.py +++ llvm/tools/opt-viewer/opt-viewer.py @@ -1,25 +1,24 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 import argparse -import errno import functools -import html -import io -from multiprocessing import cpu_count import os.path import re import shutil import sys - +import json +import glob +import pathlib +import collections +from datetime import datetime from pygments import highlight from pygments.lexers.c_cpp import CppLexer from pygments.formatters import HtmlFormatter - import optpmap import optrecord - +import config_parser +import logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") desc = '''Generate HTML output to visualize optimization records from the YAML files generated with -fsave-optimization-record and -fdiagnostics-show-hotness. @@ -35,123 +34,137 @@ context = Context() -def suppress(remark): - if remark.Name == 'sil.Specialized': - return remark.getArgDict()['Function'][0].startswith('\"Swift.') - elif remark.Name == 'sil.Inlined': - return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.')) - return False - -class SourceFileRenderer: - def __init__(self, source_dir, output_dir, filename, no_highlight): - self.filename = filename - existing_filename = None - if os.path.exists(filename): - existing_filename = filename - else: - fn = os.path.join(source_dir, filename) - if os.path.exists(fn): - existing_filename = fn - - self.no_highlight = no_highlight - self.stream = io.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8') - if existing_filename: - self.source_stream = io.open(existing_filename, encoding='utf-8') - else: - self.source_stream = None - print(u''' - -

Unable to locate file {}

- - '''.format(filename), file=self.stream) - self.html_formatter = HtmlFormatter(encoding='utf-8') - self.cpp_lexer = CppLexer(stripnl=False) +def render_file_source(source_dir, output_dir, filename, line_remarks): + html_filename = os.path.join(output_dir, optrecord.html_file_name(filename)) + filename = filename if os.path.exists(filename) else os.path.join(source_dir, filename) - def render_source_lines(self, stream, line_remarks): + html_formatter = HtmlFormatter(encoding='utf-8') + cpp_lexer = CppLexer(stripnl=False) + + def render_source_lines(stream, line_remarks): file_text = stream.read() - if self.no_highlight: - html_highlighted = file_text - else: - html_highlighted = highlight( + html_highlighted = highlight( file_text, - self.cpp_lexer, - self.html_formatter) + cpp_lexer, + html_formatter) - # Note that the API is different between Python 2 and 3. On - # Python 3, pygments.highlight() returns a bytes object, so we - # have to decode. On Python 2, the output is str but since we - # support unicode characters and the output streams is unicode we - # decode too. - html_highlighted = html_highlighted.decode('utf-8') + # Note that the API is different between Python 2 and 3. On + # Python 3, pygments.highlight() returns a bytes object, so we + # have to decode. On Python 2, the output is str but since we + # support unicode characters and the output streams is unicode we + # decode too. + html_highlighted = html_highlighted.decode('utf-8') - # Take off the header and footer, these must be - # reapplied line-wise, within the page structure - html_highlighted = html_highlighted.replace('

', '')
-            html_highlighted = html_highlighted.replace('

', '') + # Take off the header and footer, these must be + # reapplied line-wise, within the page structure + html_highlighted = html_highlighted.replace('

', '')
+        html_highlighted = html_highlighted.replace('

', '') for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1): - print(u''' - -{linenum} - - -

{html_line}

-'''.format(**locals()), file=self.stream) - - for remark in line_remarks.get(linenum, []): - if not suppress(remark): - self.render_inline_remarks(remark, html_line) - - def render_inline_remarks(self, r, line): - inlining_context = r.DemangledFunctionName - dl = context.caller_loc.get(r.Function) + yield [f'{linenum}', '', '', f'

{html_line}

', ''] + + cur_line_remarks = line_remarks.get(linenum, []) + from collections import defaultdict + d = defaultdict(list) + count_deleted = defaultdict(int) + for obj in cur_line_remarks: + if len(d[obj.Name]) < 5: + d[obj.Name].append(obj) + else: + count_deleted[obj.Name] += 1 + + for obj_name, remarks in d.items(): + # render caret line, if all rendered remarks share a column + columns = [r.Column for r in remarks] + if all(c == columns[0] for c in columns) and columns[0] != 0: + yield ['', + 0, + {'class': f"column-entry-yellow", 'text': ''}, + {'class': 'column-entry-yellow', + 'text': f'''{" "*(columns[0]-1) + '^'} '''}, + {'class': f"column-entry-yellow", 'text': ''}, + ] + for remark in remarks: + yield render_inline_remark(remark, html_line) + if count_deleted[obj_name] != 0: + yield ['', + 0, + {'class': f"column-entry-yellow", 'text': ''}, + {'class': 'column-entry-yellow', 'text': f'''...{count_deleted[obj_name]} similar remarks omitted. '''}, + {'class': f"column-entry-yellow", 'text': ''}, + ] + + + def render_inline_remark(remark, line): + inlining_context = remark.DemangledFunctionName + dl = context.caller_loc.get(remark.Function) if dl: dl_dict = dict(list(dl)) link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2) - inlining_context = "{r.DemangledFunctionName}".format(**locals()) + inlining_context = f"{remark.DemangledFunctionName}" - # Column is the number of characters *including* tabs, keep those and - # replace everything else with spaces. - indent = line[:max(r.Column, 1) - 1] - indent = re.sub('\S', ' ', indent) + start_line = re.sub("^", "", line) + spaces = len(start_line) - len(start_line.lstrip()) + indent = f"{spaces + 2}ch" # Create expanded message and link if we have a multiline message. - lines = r.message.split('\n') + lines = remark.message.split('\n') if len(lines) > 1: expand_link = '+' message = lines[0] - expand_message = u''' + other_lines = "\n".join(lines[1:]) + expand_message = f''' '''.format(indent, '\n'.join(lines[1:])) +

{other_lines}

- - - - - -'''.format(os.path.basename(self.filename)), file=self.stream) - self.render_source_lines(self.source_stream, line_remarks) - - print(u''' - -

Line -	Hotness -	Optimization -	Source -	Inline Context -

-''', file=self.stream) - - -class IndexRenderer: - def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index): - self.stream = io.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8') - self.should_display_hotness = should_display_hotness - self.max_hottest_remarks_on_index = max_hottest_remarks_on_index - - def render_entry(self, r, odd): - escaped_name = html.escape(r.DemangledFunctionName) - print(u''' - -{r.DebugLocString} -{r.RelativeHotness} -{escaped_name} -{r.PassWithDiffPrefix} -'''.format(**locals()), file=self.stream) - - def render(self, all_remarks): - print(u''' + +''') + +def render_index(output_dir, all_remarks): + def render_entry(remark): + return dict(description=remark.Name, + loc=f"{remark.DebugLocString}", + message=remark.message, + functionName=remark.DemangledFunctionName, + relativeHotness=remark.RelativeHotness, + color=remark.color) + + entries = [render_entry(remark) for remark in all_remarks] + + entries_summary = collections.Counter(e['description'] for e in entries) + entries_summary_li = '\n'.join(f"

{key}: {value}" for key, value in entries_summary.items()) + + index_path = os.path.join(output_dir, 'index.html') + with open(index_path, 'w', encoding='utf-8') as f: + f.write(f''' - + + + + + + +OptView2 Index +

{len(entries_summary)} issue types:

+{entries_summary_li} +

- - - - - - -''', file=self.stream) - - max_entries = None - if self.should_display_hotness: - max_entries = self.max_hottest_remarks_on_index - - for i, remark in enumerate(all_remarks[:max_entries]): - if not suppress(remark): - self.render_entry(remark, i % 2) - print(u''' -

Source Location

Hotness

Function

Pass

+ -''', file=self.stream) - + +''') + return index_path -def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_): +# TODO: make pmap and _wrapped_func pack arguments, so these dummies won't be needed +def _render_file(source_dir, output_dir, ctx, entry, exclude_names, exclude_text, collect_opt_success, remarks_src_dir): global context context = ctx filename, remarks = entry - SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks) + render_file_source(source_dir, output_dir, filename, remarks) def map_remarks(all_remarks): @@ -262,38 +316,54 @@ file_remarks, source_dir, output_dir, - no_highlight, should_display_hotness, - max_hottest_remarks_on_index, - num_jobs, - should_print_progress): - try: - os.makedirs(output_dir) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(output_dir): - pass - else: - raise + num_jobs=1, + open_browser=False): + pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) + + logging.info('Rendering index page...') + logging.info(f" {len(all_remarks):d} raw remarks") + if len(all_remarks) == 0: + logging.warning("Not generating report!") + return + + sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix)) + unique_lines_remarks = [sorted_remarks[0]] + for rmk in sorted_remarks: + last_unq_rmk = unique_lines_remarks[-1] + last_rmk_key = (last_unq_rmk.File, last_unq_rmk.Line, last_unq_rmk.Column, last_unq_rmk.PassWithDiffPrefix) + rmk_key = (rmk.File, rmk.Line, rmk.Column, rmk.PassWithDiffPrefix) + if rmk_key != last_rmk_key: + unique_lines_remarks.append(rmk) + logging.info(" {:d} unique source locations".format(len(unique_lines_remarks))) - if should_print_progress: - print('Rendering index page...') if should_display_hotness: - sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True) + sorted_remarks = sorted(unique_lines_remarks, key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True) else: - sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function)) - IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks) - - shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "style.css"), output_dir) - - _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight) - if should_print_progress: - print('Rendering HTML files...') - optpmap.pmap(_render_file_bound, - file_remarks.items(), - num_jobs, - should_print_progress) - + sorted_remarks = sorted(unique_lines_remarks, key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function)) + + index_path = render_index(output_dir, sorted_remarks) + + logging.info("Copying assets") + assets_path = pathlib.Path(output_dir) / "assets" + assets_path.mkdir(parents=True, exist_ok=True) + for filename in glob.glob(os.path.join(str(pathlib.Path(os.path.realpath(__file__)).parent), "assets", '*.*')): + shutil.copy(filename, assets_path) + + _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context) + logging.info('Rendering HTML files...') + optpmap.pmap(func=_render_file_bound, + iterable=file_remarks.items(), + processes=num_jobs) + + url_path = f'file://{os.path.abspath(index_path)}' + logging.info(f'Done - check the index page at {url_path}') + if open_browser: + try: + import webbrowser + webbrowser.open(url_path) + except Exception: + pass def main(): parser = argparse.ArgumentParser(description=desc) @@ -315,65 +385,118 @@ default=None, type=int, help='Max job count (defaults to %(default)s, the current CPU count)') + parser.add_argument( '--source-dir', '-s', default='', help='set source directory') + parser.add_argument( - '--no-progress-indicator', - '-n', - action='store_true', - default=False, - help='Do not display any indicator of how many YAML files were read ' - 'or rendered into HTML.') + '--demangler', + help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler) + parser.add_argument( - '--max-hottest-remarks-on-index', - default=1000, - type=int, - help='Maximum number of the hottest remarks to appear on the index page') + '--exclude-name', + default='', + help='Omit optimization remarks with names matched by this regex') + + parser.add_argument( + '--exclude-text', + default='', + help='Omit optimization remarks with names matched by this regex') + parser.add_argument( - '--no-highlight', + '--collect-opt-success', action='store_true', - default=False, - help='Do not use a syntax highlighter when rendering the source code') + help='Collect all optimization remarks, not just failures') + parser.add_argument( - '--demangler', - help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler) + '--annotate-external', + action='store_true', + help='Annotate all files, including system headers') parser.add_argument( - '--filter', - default='', - help='Only display remarks from passes matching filter expression') + '--open-browser', + action='store_true', + help='Open browser after generating HTML files') + + parser.add_argument( + '--split-top-folders', + action='store_true', + help='Operate separately on every top level subfolder containing opt files - to workaround out-of-memory crashes') # Do not make this a global variable. Values needed to be propagated through # to individual classes and functions to be portable with multiprocessing across # Windows and non-Windows. + cur_folder = pathlib.Path(__file__).parent.resolve() + conf_file = os.path.join(cur_folder, "config.yaml") + with open(conf_file, 'r') as config_file: + config = config_parser.parse(config_file) + parser.set_defaults(**config) args = parser.parse_args() - print_progress = not args.no_progress_indicator if args.demangler: optrecord.Remark.set_demangler(args.demangler) - files = optrecord.find_opt_files(*args.yaml_dirs_or_files) - if not files: - parser.error("No *.opt.yaml files found") - sys.exit(1) - - all_remarks, file_remarks, should_display_hotness = \ - optrecord.gather_results(files, args.jobs, print_progress, args.filter) - - map_remarks(all_remarks) - - generate_report(all_remarks, - file_remarks, - args.source_dir, - args.output_dir, - args.no_highlight, - should_display_hotness, - args.max_hottest_remarks_on_index, - args.jobs, - print_progress) + start_time = datetime.now() + + if args.split_top_folders: + subfolders = [] + for item in os.listdir(args.yaml_dirs_or_files[0]): + if os.path.isfile(os.path.join(args.yaml_dirs_or_files[0], item)): + continue + subfolders.append(item) + + for subfolder in subfolders: + files = optrecord.find_opt_files(os.path.join(args.yaml_dirs_or_files[0], subfolder)) + if not files: + continue + + logging.info(f"Processing subfolder {subfolder}") + + all_remarks, file_remarks, should_display_hotness = \ + optrecord.gather_results(filenames=files, num_jobs=args.jobs, + exclude_names=args.exclude_names, + exclude_text=args.exclude_text, + collect_opt_success=args.collect_opt_success, + annotate_external=args.annotate_external) + + map_remarks(all_remarks) + + generate_report(all_remarks=all_remarks, + file_remarks=file_remarks, + source_dir=args.source_dir, + output_dir=os.path.join(args.output_dir, subfolder), + should_display_hotness=should_display_hotness, + num_jobs=args.jobs, + open_browser=args.open_browser) + else: # not split_top_foders + files = optrecord.find_opt_files(os.path.join(*args.yaml_dirs_or_files)) + if not files: + parser.error("No *.opt.yaml files found") + sys.exit(1) + + all_remarks, file_remarks, should_display_hotness = \ + optrecord.gather_results(filenames=files, num_jobs=args.jobs, + exclude_names=args.exclude_names, + exclude_text=args.exclude_text, + collect_opt_success=args.collect_opt_success, + annotate_external=args.annotate_external) + + map_remarks(all_remarks) + + generate_report(all_remarks=all_remarks, + file_remarks=file_remarks, + source_dir=args.source_dir, + output_dir=args.output_dir, + should_display_hotness=should_display_hotness, + num_jobs=args.jobs, + open_browser=args.open_browser) + + end_time = datetime.now() + logging.info(f"Ran for {end_time-start_time}") if __name__ == '__main__': main() + Index: llvm/tools/opt-viewer/optpmap.py =================================================================== --- llvm/tools/opt-viewer/optpmap.py +++ llvm/tools/opt-viewer/optpmap.py @@ -14,43 +14,40 @@ def _wrapped_func(func_and_args): - func, argument, should_print_progress, filter_ = func_and_args + func, argument, exclude_names, exclude_text, collect_opt_success, annotate_external = func_and_args - if should_print_progress: - with _current.get_lock(): - _current.value += 1 - sys.stdout.write('\r\t{} of {}'.format(_current.value, _total.value)) - sys.stdout.flush() + with _current.get_lock(): + _current.value += 1 + sys.stdout.write('\r\t{} of {}'.format(_current.value, _total.value)) + sys.stdout.flush() - return func(argument, filter_) + return func(argument, exclude_names, exclude_text, collect_opt_success, annotate_external) -def pmap(func, iterable, processes, should_print_progress, filter_=None, *args, **kwargs): +def pmap(func, iterable, processes, exclude_names=None, exclude_text=None, collect_opt_success=False, annotate_external=False, *args, **kwargs): """ A parallel map function that reports on its progress. Applies `func` to every item of `iterable` and return a list of the results. If `processes` is greater than one, a process pool is used to run - the functions in parallel. `should_print_progress` is a boolean value that - indicates whether a string 'N of M' should be printed to indicate how many - of the functions have finished being run. + the functions in parallel. """ global _current global _total _current = multiprocessing.Value('i', 0) _total = multiprocessing.Value('i', len(iterable)) - func_and_args = [(func, arg, should_print_progress, filter_) for arg in iterable] + func_and_args = [(func, arg, exclude_names, exclude_text, collect_opt_success, annotate_external) for arg in iterable] if processes == 1: result = list(map(_wrapped_func, func_and_args, *args, **kwargs)) else: pool = multiprocessing.Pool(initializer=_init, initargs=(_current, _total,), - processes=processes) + processes=processes, + maxtasksperchild=2) result = pool.map(_wrapped_func, func_and_args, *args, **kwargs) pool.close() pool.join() - if should_print_progress: - sys.stdout.write('\r') + sys.stdout.write('\n') return result Index: llvm/tools/opt-viewer/optrecord.py =================================================================== --- llvm/tools/opt-viewer/optrecord.py +++ llvm/tools/opt-viewer/optrecord.py @@ -1,14 +1,19 @@ #!/usr/bin/env python -from __future__ import print_function +import logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") import io import yaml -# Try to use the C parser. +import pickle + +import optrecord + try: + # Try to use the C parser. from yaml import CLoader as Loader except ImportError: - print("For faster parsing, you may want to install libYAML for PyYAML") + logging.warning("For faster parsing, you may want to install libYAML for PyYAML") from yaml import Loader import html @@ -57,7 +62,7 @@ # Work-around for http://pyyaml.org/ticket/154. yaml_loader = Loader - default_demangler = 'c++filt -n' + default_demangler = 'c++filt -n -p' demangler_proc = None @classmethod @@ -268,24 +273,39 @@ class Failure(Missed): yaml_tag = '!Failure' -def get_remarks(input_file, filter_=None): +def get_remarks(input_file, exclude_names=None, exclude_text = None, collect_opt_success=False, annotate_external=False): max_hotness = 0 all_remarks = dict() file_remarks = defaultdict(functools.partial(defaultdict, list)) + #logging.debug(f"Parsing {input_file}") + #TODO: filter unique name+file+line loc *here* with io.open(input_file, encoding = 'utf-8') as f: docs = yaml.load_all(f, Loader=Loader) - filter_e = None - if filter_: - filter_e = re.compile(filter_) + exclude_text_e = None + if exclude_text: + exclude_text_e = re.compile(exclude_text) + exclude_names_e = None + if exclude_names: + exclude_names_e = re.compile(exclude_names) for remark in docs: remark.canonicalize() # Avoid remarks withoug debug location or if they are duplicated if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: continue - if filter_e and not filter_e.search(remark.Pass): + if not collect_opt_success and not (isinstance(remark, optrecord.Missed) | isinstance(remark, optrecord.Failure)): + continue + + if not annotate_external: + if os.path.isabs(remark.File): + continue + + if exclude_names_e and exclude_names_e.search(remark.Name): + continue + + if exclude_text_e and exclude_text_e.search(remark.message): continue all_remarks[remark.key] = remark @@ -302,13 +322,23 @@ return max_hotness, all_remarks, file_remarks -def gather_results(filenames, num_jobs, should_print_progress, filter_=None): - if should_print_progress: - print('Reading YAML files...') +def gather_results(filenames, num_jobs, annotate_external=False, exclude_names=None, exclude_text=None, collect_opt_success=False): + logging.info('Reading YAML files...') + load = False if not Remark.demangler_proc: Remark.set_demangler(Remark.default_demangler) - remarks = optpmap.pmap( - get_remarks, filenames, num_jobs, should_print_progress, filter_) + if not load: + remarks = optpmap.pmap( + get_remarks, filenames, num_jobs, exclude_names, exclude_text, collect_opt_success, annotate_external) + #TODO: pass output dir + #logging.info("saving remarks") + #with open(os.path.join("/home/ofek/", "remarks"), 'wb') as remarks_file: + # pickle.dump(remarks, remarks_file, pickle.HIGHEST_PROTOCOL) + else: + with open(os.path.join("/home/ofek/", "remarks"), 'rb') as remarks_file: + remarks = pickle.load(remarks_file) + + # assert (remarks == remarks1) max_hotness = max(entry[0] for entry in remarks) def merge_file_remarks(file_remarks_job, all_remarks, merged): Index: llvm/tools/opt-viewer/requirements.txt =================================================================== --- /dev/null +++ llvm/tools/opt-viewer/requirements.txt @@ -0,0 +1,3 @@ +PyYAML>=3.12 +Pygments>=2.2.0 +#guppy>=0.1.11

Unable to locate file {}

Unable to locate file {filename}

{os.path.abspath(filename)}

{len(entries_summary)} issue types: