Add geomean summary row to output from utils/compare.py.

[fhahn]

Summary

This change adds a summary row with the geometric mean of the selected
metric to the output of utils/compare.py. The 'rhs' values are
normalized to the 'lhs' values and the geometric mean of the results
is computed.

This should be similar to the geomean row in the LNT HTML UI and allow
to compare 2 sets of runs.

[fhahn]

Ping

[fhahn]

ping

[anemet]

Seems reasonable.

test-suite/trunk/utils/compare.py

 #!/usr/bin/env python
 """Tool to filter, organize, compare and display benchmarking results. Usefull
 for smaller datasets. It works great with a few dozen runs it is not designed to
 deal with hundreds.
 Requires the pandas library to be installed."""
 from __future__ import print_function
 
 import pandas as pd
+from scipy import stats
 import sys
 import os.path
 import re
 import numbers
 import argparse
 
 def read_lit_json(filename):
     import json
for filename in filenames:
         if prev_index is not None and prev_index.name != data.index.name:
             sys.stderr.write("Warning: Mismatched index names: '%s' vs '%s'\n"
                              % (prev_index.name, data.index.name))
         prev_index = data.index
     # Merge datasets
     d = pd.concat(datasets, axis=0, names=['run'], keys=datasetnames)
     return d
 
-def add_diff_column(d, absolute_diff=False):
-    values = d.unstack(level=0)
-
-    has_two_runs = d.index.get_level_values(0).nunique() == 2
+def get_values(values):
+    # Create data view without diff column.
+    if 'diff' in values.columns:
+        values = values[[c for c in values.columns if c != 'diff']]
+    has_two_runs = len(values.columns) == 2
     if has_two_runs:
-        values0 = values.iloc[:,0]
-        values1 = values.iloc[:,1]
+        return (values.iloc[:,0], values.iloc[:,1])
     else:
-        values0 = values.min(axis=1)
-        values1 = values.max(axis=1)
+        return (values.min(axis=1), values.max(axis=1))
 
+def add_diff_column(values, absolute_diff=False):
+    values0, values1 = get_values(values)
     # Quotient or absolute difference?
     if absolute_diff:
         values['diff'] = values1 - values0
     else:
         values['diff'] = values1 / values0
         values['diff'] -= 1.0
-    # unstack() gave us a complicated multiindex for the columns, simplify
-    # things by renaming to a simple index.
-    values.columns = [(c[1] if c[1] else c[0]) for c in values.columns.values]
     return values
 
+def add_geomean_row(data, dataout):
+    """
+    Normalize values1 over values0, compute geomean difference and add a
+    summary row to dataout.
+    """
+    values0, values1 = get_values(data)
+    relative = values1 / values0
+    gm_diff = stats.gmean(relative) - 1.0
+
+    gm_row = {c: '' for c in dataout.columns}
+    gm_row['diff'] = gm_diff
+    gm_row['Program'] = 'Geomean difference'
+    return dataout.append(gm_row, ignore_index=True)
+
 def filter_failed(data, key='Exec'):
     return data.loc[data[key] == "pass"]
 
 def filter_short(data, key='Exec_Time', threshold=0.6):
     return data.loc[data[key] >= threshold]
 
 def filter_same_hash(data, key='hash'):
     assert key in data.columns
def print_result(d, limit_output=True, shorten_names=True,
     dataout = d
     if limit_output:
         # Take 15 topmost elements
         dataout = dataout.head(15)
 
     # Turn index into a column so we can format it...
     dataout.insert(0, 'Program', dataout.index)
 
+    if show_diff_column:
+        dataout = add_geomean_row(d, dataout)
+
     formatters = dict()
     formatters['diff'] = format_diff
     if shorten_names:
         drop_prefix, drop_suffix = determine_common_prefix_suffix(dataout.Program)
         def format_name(name, common_prefix, common_suffix):
             name = name[common_prefix:]
             if common_suffix > 0:
                 name = name[:-common_suffix]
             return "%-45s" % truncate(name, 10, 30)
 
         formatters['Program'] = lambda name: format_name(name, drop_prefix, drop_suffix)
-    float_format = lambda x: "%6.2f" % (x,)
+    def float_format(x):
+        if x == '':
+            return ''
+        return "%6.2f" % (x,)
+
     pd.set_option("display.max_colwidth", 0)
     out = dataout.to_string(index=False, justify='left',
                             float_format=float_format, formatters=formatters)
     print(out)
     print(d.describe())
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(prog='compare.py')
if __name__ == "__main__":
     final_size = len(data.groupby(level=1))
     if final_size != initial_size:
         print("Remaining: %d" % (final_size,))
 
     # Reduce / add columns
     print("Metric: %s" % (",".join(metrics),))
     if len(metrics) > 0:
         data = data[metrics]
+
+    data = data.unstack(level=0)
+    # unstack() gave us a complicated multiindex for the columns, simplify
+    # things by renaming to a simple index.
+    data.columns = [(c[1] if c[1] else c[0]) for c in data.columns.values]
+
     data = add_diff_column(data)
 
     sortkey = 'diff'
     if len(config.files) == 1:
         sortkey = data.columns[0]
 
     # Print data
     print("")
     shorten_names = not config.full
     limit_output = (not config.all) and (not config.full)
     print_result(data, limit_output, shorten_names, config.show_diff, sortkey)