utils/compare.py: Add --absolute-diff

[arichardson]

Summary

This is useful for metrics such as number of times a given pass performed
a transformation, etc.

[MatzeB]

LGTM

utils/compare.py

294–295

Should this have a default=False? (I guess currently you get None which works but feels a bit accidental)

[arichardson]

utils/compare.py

294–295

Good question, I just checked and it turns out the store_true action sets the default to False:

>>> import argparse
>>> parser = argparse.ArgumentParser(prog='compare.py')
>>> parser.add_argument('--absolute-diff', action='store_true', help='Use an absolute instead of a relative difference')
_StoreTrueAction(option_strings=['--absolute-diff'], dest='absolute_diff', nargs=0, const=True, default=False, type=None, choices=None, help='Use an absolute instead of a relative difference', metavar=None)
>>> parser.parse_args()
Namespace(absolute_diff=False)

utils/compare.py

def get_values(values):
     has_two_runs = len(values.columns) == 2
     if has_two_runs:
         return (values.iloc[:,0], values.iloc[:,1])
     else:
         return (values.min(axis=1), values.max(axis=1))
 
 def add_diff_column(metric, values, absolute_diff=False):
     values0, values1 = get_values(values[metric])
+    values0.fillna(0.0, inplace=True)
+    values1.fillna(0.0, inplace=True)
     # Quotient or absolute difference?
     if absolute_diff:
         values[(metric, 'diff')] = values1 - values0
     else:
         values[(metric, 'diff')] = (values1 / values0) - 1.0
     return values
 
 def add_geomean_row(metrics, data, dataout):
for name in names:
             suffix = name0[-suffix_len:]
 
     if suffix[0] != '.' and suffix[0] != '_':
         suffix_len = 0
     suffix_len = max(0, min(shortest_name - prefix_len - min_len, suffix_len))
     prefix_len = max(0, min(shortest_name - suffix_len, prefix_len))
     return (prefix_len, suffix_len)
 
-def format_diff(value):
+def format_relative_diff(value):
     if not isinstance(value, numbers.Integral):
         return "%4.1f%%" % (value * 100.)
     else:
         return "%-5d" % value
 
 def print_result(d, limit_output=True, shorten_names=True, minimal_names=False,
-                 show_diff_column=True, sortkey='diff', sort_by_abs=True):
+                 show_diff_column=True, sortkey='diff', sort_by_abs=True,
+                 absolute_diff=False):
     metrics = d.columns.levels[0]
     if sort_by_abs:
         d = d.sort_values(by=(metrics[0], sortkey), key=pd.Series.abs, ascending=False)
     else:
         d = d.sort_values(by=(metrics[0], sortkey), ascending=False)
 
     # Ensure that the columns are grouped by metric (rather than having the
     # diffs at the end of the line).
     d = d.reindex(columns=d.columns.levels[0], level=0)
 
     if not show_diff_column:
         # Remove all diff columns (using level=1 since level 0 is the metric).
         d.drop(labels='diff', level=1, axis=1, inplace=True)
     dataout = d
     if limit_output:
         # Take 15 topmost elements
         dataout = dataout.head(15)
 
     formatters = dict()
+    if not absolute_diff:
         for m in metrics:
-        formatters[(m, 'diff')] = format_diff
+            formatters[(m, 'diff')] = format_relative_diff
     # Turn index into a column so we can format it...
     formatted_program = dataout.index.to_series()
     if shorten_names:
         def format_name(name, common_prefix, common_suffix):
             name = name[common_prefix:]
             if common_suffix > 0:
                 name = name[:-common_suffix]
             return "%-45s" % truncate(name, 10, 30)
if shorten_names:
         if minimal_names:
             formatted_program = formatted_program.map(strip_name_fully)
         else:
             drop_prefix, drop_suffix = determine_common_prefix_suffix(formatted_program)
             formatted_program = formatted_program.map(lambda name: format_name(name, drop_prefix, drop_suffix))
     dataout.insert(0, 'Program', formatted_program)
     # Add the geometric mean row after we have formatted the program names
     # as it will otherwise interfere with common prefix/suffix computation.
-    if show_diff_column:
+    if show_diff_column and not absolute_diff:
+        # geometric mean only makes sense for relative differences.
         dataout = add_geomean_row(metrics, d, dataout)
 
     def float_format(x):
         if x == '':
             return ''
         return "%6.2f" % (x,)
 
     pd.set_option("display.max_colwidth", 0)
     pd.set_option('display.width', 0)
     # Print an empty value instead of NaN (for the geomean row).
     out = dataout.to_string(index=False, justify='left', na_rep='',
                             float_format=float_format, formatters=formatters)
     print(out)
     print(d.describe())
 
 def main():
     parser = argparse.ArgumentParser(prog='compare.py')
     parser.add_argument('-a', '--all', action='store_true')
     parser.add_argument('-f', '--full', action='store_true')
     parser.add_argument('-m', '--metric', action='append', dest='metrics',
                         default=[])
     parser.add_argument('--nodiff', action='store_false', dest='show_diff',
                         default=None)
     parser.add_argument('--diff', action='store_true', dest='show_diff')
+    parser.add_argument('--absolute-diff', action='store_true',
+                        help='Use an absolute instead of a relative difference')
     parser.add_argument('--filter-short', action='store_true',
                         dest='filter_short')
     parser.add_argument('--no-filter-failed', action='store_false',
                         dest='filter_failed', default=True)
     parser.add_argument('--filter-hash', action='store_true',
                         dest='filter_hash', default=False)
     parser.add_argument('--filter-blacklist',
                         dest='filter_blacklist', default=None)
def main():
     # Reduce / add columns
     print("Metric: %s" % (",".join(metrics),))
     if len(metrics) > 0:
         data = data[metrics]
 
     data = data.unstack(level=0)
 
     for metric in data.columns.levels[0]:
-        data = add_diff_column(metric, data)
+        data = add_diff_column(metric, data, absolute_diff=config.absolute_diff)
 
     sortkey = 'diff'
     # TODO: should we still be sorting by diff even if the diff is hidden?
     if len(config.files) == 1:
         sortkey = data.columns.levels[1][0]
 
     # Print data
     print("")
     shorten_names = not config.full
     limit_output = (not config.all) and (not config.full)
-    print_result(data, limit_output, shorten_names, config.minimal_names, config.show_diff, sortkey, config.no_abs_sort)
+    print_result(data, limit_output, shorten_names, config.minimal_names,
+                 config.show_diff, sortkey, config.no_abs_sort,
+                 config.absolute_diff)
 
 
 if __name__ == "__main__":
     main()