LNT graph New Setting options to use mean as aggregation function

[Andi]

Summary

Added a new settings option in LNT graph to use the mean instead of min/max as the aggregation function.

Added that this setting persists for the duration of the session.

Added simple test cases

[cmatthews]

This is a much needed update to the graphing. Thanks Andi!

lnt/server/ui/templates/v4_graph.html

160 ↗

(On Diff #80506)

Maybe just "Mean() as aggregation:"

lnt/server/ui/views.py

515 ↗

(On Diff #80506)

Combine with line above.

lnt/util/stats.py

28 ↗

(On Diff #80506)

I think this needs to work with ints as well as floats.

28 ↗

(On Diff #80506)

Better to have this in the docstring. Comments should be full sentences (missing .).

29 ↗

(On Diff #80506)

A docstring describing the input for this is needed. It is an enumeration of (val, index) tuples?

31 ↗

(On Diff #80506)

space after ,

38 ↗

(On Diff #80506)

Should this be 0, or some index value derived from the input pairs?

38 ↗

(On Diff #80506)

space around operator

tests/server/ui/V4Pages.py

439 ↗

(On Diff #80506)

Why no slash in front of these?

tests/server/ui/statsTester.py

4 ↗

(On Diff #80506)

Probably no need to add this.

11 ↗

(On Diff #80506)

Could you refactor this to use the Python unittest framework? I am trying to use that everywhere for the Python based tests.

13 ↗

(On Diff #80506)

I think logging does not need to be in this test. We normally only enable that for sever tests that might log something.

26 ↗

(On Diff #80506)

spaces after ,

26 ↗

(On Diff #80506)

Test with floats and ints.

27 ↗

(On Diff #80506)

Space after ,

30 ↗

(On Diff #80506)

You mention it should work with enumerations, maybe should be tested with one.

[Andi]

lnt/util/stats.py

38 ↗

(On Diff #80506)

The aggregation function returns two values: agg_value, agg_index

For min and max, the index makes sense as it corresponds to an actual data point's value. Then the agg_index is used to populate metadata with the run_id and date from where this value came from.

The mean, is not really associated with any specific data point, but rather with all the data points coming form a specific date and run (as far as I can tell). So I picked zero to get the data and run_id.

[cmatthews]

lnt/util/stats.py

38 ↗

(On Diff #80506)

Okay, that makes sense.

[Andi]

Updated the code in accordance with the reviewer's comments.
Also changed the statsTester to use unittests

[cmatthews]

Looks good. Thanks Andi!

lnt/trunk/lnt/server/ui/templates/v4_graph.html

function init_graph() {
   var graph_options = {
       series : {
         lines : {
           lineWidth : 1 },
         shadowSize : 0
         },
       highlight : {
 {% if revision_range is not none %}
-        range: {{revision_range|tojson|safe}} 
+        range: {{revision_range|tojson|safe}}
 {% else %}
         enabled: false
 {% endif %}
       },
       zoom : { interactive : false },
       pan : { interactive : true,
               frameRate: 60 },
       grid : {
         hoverable : true,
         clickable: true }
       };
 
   // Add baseline lines
   graph_options['grid']['markings'] = baseline_plots;
   var tmp_plots = update_graphplots(graph_plots);
   var main_plot = $.plot(graph, tmp_plots, graph_options);
-  
   // Add tooltips.
   graph.bind("plotclick", function (e, p, i) {
       update_tooltip(e, p, i, show_tooltip, tmp_plots);
   });
 
   // Set up the overview graph.
   var overview = $("#overview")
   var overview_plots = {{overview_plots|tojson|safe}};
$("#overview").bind("plotselected", function (event, ranges) {
     // Set the zooming on the plot.
     $.plot(graph, graph_plots,
       $.extend(true, {}, graph_options, {
         xaxis: { min: ranges.xaxis.from, max: ranges.xaxis.to },
         yaxis: { min: ranges.yaxis.from, max: ranges.yaxis.to }
       }));
   });
   bind_zoom_bar(main_plot);
-  
 }
 
 
 function init_page() {
     // First start the requests for regrssion data.
     var urls = $(".data-row").each(function (index, val) {
         $.getJSON(get_api_url("regression",
                               db_name,
<td>
         <a class="dropdown-toggle" data-toggle="dropdown" href="#settings">
           Settings <i class="icon-wrench"></i>
         </a>
         <div class="dropdown-menu">
           <form action="" method="get">
           <table class="table table-striped table-hover table-condensed">
             <tbody>
               <tr>
+                <td>Mean() as Aggregation</td>
+                <td><input type="checkbox" name="switch_min_mean" value="yes"
+                    {{ 'checked' if options.switch_min_mean else ""}}></td>
+              </tr>
+              <tr>
                 <td>Hide Line Plot:</td>
                 <td><input type="checkbox" name="hide_lineplot" value="yes"
                      {{ 'checked' if options.hide_lineplot else ""}}></td>
               </tr>
               <tr>
                 <td>Show Median Absolute Deviation:</td>
                 <td><input type="checkbox" name="show_mad" value="yes"
                      {{ 'checked' if options.show_mad else ""}}></td>
<td>
           {% endif %}
           {% if name.startswith('baseline.') %}
           <input type="hidden" name="{{name}}" value="{{value}}">
           {% endif %}
           {% if name == 'mean' %}
           <input type="hidden" name="{{name}}" value="{{value}}">
           {% endif %}
           {% endfor %}
-            
           <input class="btn btn-primary" style="clear: left; width: 100%"
                  type="submit" name="submit" value="Update" />
           </form>
         </div>
       </div>
     </td>
   </tr>
 
   <tr>
         <td colspan="2">
             <div id="graphbox" style="margin-left:20px">
                 <div id="graph" style="height:500px"></div>
                 <div id="zoombar" style="width:40px;">
                     <button id="in" type="button" class="btn btn-default" style="width:100%;text-align:center;">+</button>
                     <br>
                     <button id="out" type="button" class="btn btn-default" style="width:100%; text-align:center;">-</button>
                 </div>
                 <div id="yaxis">Metric</div>
                 <div id="xaxis">Order</div>
             </div>
         </td>
     </tr>
-  
     <tr>
         <td colspan="2">
             <div id="overview" style="height:80px;margin-top:20px;margin-left:20px"></div>
         </td>
     </td>
 
 </table>
-  
   <h3>Legend</h3>
   <table class="table table-condensed table-hover table-striped">
     <tr>
       <th></th>
       <th>Machine</th>
       <th>Test</th>
       <th>Type</th>
     </tr>

lnt/trunk/lnt/server/ui/views.py

 import datetime
 import os
 import re
 import tempfile
 import time
 import copy
 import json
 
 import flask
+from flask import session
 from flask import abort
 from flask import current_app
 from flask import g
 from flask import make_response
 from flask import redirect
 from flask import render_template
 from flask import request
 from flask import url_for
if request.args.get('json'):
                                          run.start_time.isoformat(), run.end_time.isoformat()))
         return flask.jsonify(**json_obj)
     try:
         return render_template("v4_machine.html",
                            testsuite_name=g.testsuite_name, id=id,
                            associated_runs=associated_runs)
     except NoResultFound as e:
         abort(404)
-        
 class V4RequestInfo(object):
     def __init__(self, run_id, only_html_body=True):
         self.db = request.get_db()
         self.ts = ts = request.get_testsuite()
         self.run = run = ts.query(ts.Run).filter_by(id=run_id).first()
         if run is None:
             abort(404)
 
def __init__(self, run_id, only_html_body=True):
             'body': '', 'td': '',
             'h1': 'font-size: 14pt',
             'table': 'width: initial; font-size: 9pt;',
             'th': 'text-align: center;'
         }
         classes = {
             'table': 'table table-striped table-condensed table-hover'
         }
-        
         reports = lnt.server.reporting.runs.generate_run_report(
             self.run, baseurl=db_url_for('index', _external=True),
             only_html_body=only_html_body, result=None,
             compare_to=self.compare_to, baseline=self.baseline,
             num_comparison_runs=self.num_comparison_runs,
             aggregation_fn=self.aggregation_fn, confidence_lv=confidence_lv,
             styles=styles, classes=classes)
         _, self.text_report, self.html_report, self.sri = reports

 @v4_route("/graph")
 def v4_graph():
     from lnt.server.ui import util
     from lnt.testing import PASS
     from lnt.util import stats
     from lnt.external.stats import stats as ext_stats
 
     ts = request.get_testsuite()
+    switch_min_mean_local = False
 
+    if 'switch_min_mean_session' not in session:
+        session['switch_min_mean_session'] = False
     # Parse the view options.
-    options = {}
+    options = {'min_mean_checkbox': 'min()'}
+    if 'submit' in request.args:  # user pressed a button
+        if 'switch_min_mean' in request.args:  # user checked mean() checkbox
+            session['switch_min_mean_session'] = options['switch_min_mean'] = \
+                bool(request.args.get('switch_min_mean'))
+            switch_min_mean_local = session['switch_min_mean_session']
+        else:  # mean() check box is not checked
+            session['switch_min_mean_session'] = options['switch_min_mean'] = \
+                bool(request.args.get('switch_min_mean'))
+            switch_min_mean_local = session['switch_min_mean_session']
+    else:  # new page was loaded by clicking link, not submit button
+        options['switch_min_mean'] = switch_min_mean_local = \
+            session['switch_min_mean_session']
+
     options['hide_lineplot'] = bool(request.args.get('hide_lineplot'))
     show_lineplot = not options['hide_lineplot']
     options['show_mad'] = show_mad = bool(request.args.get('show_mad'))
     options['show_stddev'] = show_stddev = bool(request.args.get('show_stddev'))
     options['hide_all_points'] = hide_all_points = bool(
         request.args.get('hide_all_points'))
     options['show_linear_regression'] = show_linear_regression = bool(
         request.args.get('show_linear_regression'))
for name, data, col, field, url in graph_datum:
 
             # When we can, map x-axis to revisions, but when that is too hard
             # use the position of the sample instead.
             rev_x = convert_revision(point_label)
             x = rev_x[0] if len(rev_x)==1 else pos
 
             values = [v*normalize_by for v in data]
             aggregation_fn = min
+
+            if switch_min_mean_local:
+                aggregation_fn = lnt.util.stats.agg_mean
             if field.bigger_is_better:
                 aggregation_fn = max
+
             agg_value, agg_index = \
                 aggregation_fn((value, index)
                                for (index, value) in enumerate(values))
 
             # Generate metadata.
             metadata = {"label": point_label}
             metadata["date"] = str(dates[agg_index])
             if runs:
                 metadata["runID"] = str(runs[agg_index])
-            
             if len(graph_datum) > 1:
                 # If there are more than one plot in the graph, also label the
                 # test name.
                 metadata["test_name"] = name
 
             pts.append((x, agg_value, metadata))
 
             # Add the individual points, if requested.
             # For each point add a text label for the mouse over.
             if not hide_all_points:
                 for i,v in enumerate(values):
                     point_metadata = dict(metadata)
                     point_metadata["date"] = str(dates[i])
                     points_data.append((x, v, point_metadata))
-            
             # Add the standard deviation error bar, if requested.
             if show_stddev:
                 mean = stats.mean(values)
                 sigma = stats.standard_deviation(values)
                 errorbar_data.append((x, mean, sigma))
 
             # Add the MAD error bar, if requested.
             if show_mad:
def health():
     """Our instnace health. If queue is too long or we use too much mem,
     return 500.  Monitor might reboot us for this."""
     explode = False
     msg = "Ok"
     queue_length = async_ops.check_workers(False)
     if queue_length > 10:
         explode = True
         msg = "Queue too long."
-    
     import resource
     stats = resource.getrusage(resource.RUSAGE_SELF)
     mem = stats.ru_maxrss
     if mem > 1024**3:
         explode = True
         msg = "Over memory " + str(mem) + ">" + str(1024**3)
     if explode:
         return msg, 500

 @v4_route("/matrix", methods=['GET', 'POST'])
 def v4_matrix():
     """A table view for Run sample data, because *some* people really
     like to be able to see results textually.
     request.args.limit limits the number of samples.
     for each dataset to add, there will be a "plot.n=.m.b.f" where m is machine
     ID, b is benchmark ID and f os field kind offset. "n" is used to unique
     the paramters, and is ignored.
-    
     """
     ts = request.get_testsuite()
     # Load the matrix request parameters.
     form = MatrixOptions(request.form)
     if request.method == 'POST':
         post_limit = form.limit.data
     else:
         post_limit = MATRIX_LIMITS[0][0]
for req in data_parameters:
             .filter(req.field.column != None) \
             .order_by(ts.Order.llvm_project_revision.desc())
 
         limit = request.args.get('limit', post_limit)
         if limit or post_limit:
             limit = int(limit)
             if limit != -1:
                 q = q.limit(limit)
-            
         req.samples = defaultdict(list)
-        
         for s in q.all():
             req.samples[s[1]].append(s[0])
             all_orders.add(s[1])
             order_to_id[s[1]] = s[2]
         req.derive_stat = {}
         for order, samples in req.samples.items():
             req.derive_stat[order] = mean(samples)
     if not all_orders:
class FakeOptions(object):
         show_small_diff = False
         show_previous = False
         show_all = True
         show_delta = False
         show_stddev = False
         show_mad = False
         show_all_samples = False
         show_sample_counts = False
-        
     return render_template("v4_matrix.html",
                            testsuite_name=g.testsuite_name,
                            associated_runs=data_parameters,
                            orders=all_orders,
                            options=FakeOptions(),
                            analysis=lnt.server.reporting.analysis,
                            geomeans=order_to_geomean,
                            order_to_id=order_to_id,
                            form=form,
                            baseline_rev=baseline_rev,
                            machine_name_common=machine_name_common,
                            machine_id_common=machine_id_common,
                            order_to_date=order_to_date)

lnt/trunk/lnt/util/stats.py

else:
         return max(l)
 
 def mean(l):
     if l:
         return sum(l)/len(l)
     else:
         return None
 
+def agg_mean(pairs):
+    """Aggregation function in views.py receives input via enumerate and
+       produces a tuple.
+       Input: (value, index)
+       Output: (mean, 0), or (None, None) on invalid input.
+    """
+    if not pairs:
+        return (None, None)
+    my_sum = 0.0
+    counter = 0
+    for item in pairs:
+        my_sum += item[0]
+        counter += 1
+    if counter > 0:
+        return (my_sum / counter, 0)
+    return (None, None)
+
 
 def median(l):
     if not l:
         return None
     l = list(l)
     l.sort()
     N = len(l)
     return (l[(N-1)//2] + l[N//2])*.5

lnt/trunk/tests/server/ui/V4Pages.py

def main():
     # Don't catch out exceptions.
     app.testing = True
 
     # Create a test client.
     client = app.test_client()
 
     # Fetch the index page.
     check_code(client, '/')
-    
     # Rules the index page.
     check_code(client, '/rules')
 
     # Get the V4 overview page.
     check_code(client, '/v4/nts/')
 
     # Get a machine overview page.
     check_code(client, '/v4/nts/machine/1')
def main():
     check_code(client, '/v4/nts/graph?mean=1.2')
     # Don't crash when requesting non-existing data
     check_code(client, '/v4/nts/graph?mean=9999.2',
                expected_code=HTTP_NOT_FOUND)
     check_code(client, '/v4/nts/graph?mean=1.9999',
                expected_code=HTTP_NOT_FOUND)
     #  Check baselines work.
     check_code(client, '/v4/nts/graph?plot.0=1.3.2&baseline.60=3')
-    
     # Check some variations of the daily report work.
     check_code(client, '/v4/nts/daily_report/2012/4/12')
     check_code(client, '/v4/nts/daily_report/2012/4/11')
     check_code(client, '/v4/nts/daily_report/2012/4/13')
     check_code(client, '/v4/nts/daily_report/2012/4/10')
     check_code(client, '/v4/nts/daily_report/2012/4/14')
     check_redirect(client, '/v4/nts/daily_report',
                    '/v4/nts/daily_report/\d+/\d+/\d+$')
def main():
     check_code(client, '/v4/compile/graph?plot.3=2.3.9')
 
     # Get the mean graph page.
     check_code(client, 'v4/compile/graph?mean=2.9')
 
     # Check some variations of the daily report work.
     check_code(client, '/v4/compile/daily_report/2014/6/5?day_start=16')
     check_code(client, '/v4/compile/daily_report/2014/6/4')
-    
     check_redirect(client, '/v4/nts/regressions/new_from_graph/1/1/1/1', '/v4/nts/regressions/1')
     check_code(client, '/v4/nts/regressions/')
     check_code(client, '/v4/nts/regressions/?machine_filter=machine2')
     check_code(client, '/v4/nts/regressions/?machine_filter=machine0')
 
     check_code(client, '/v4/nts/regressions/1')
-    
     check_json(client, '/v4/nts/regressions/1?json=True')
-    
-    
-
+    # Make sure the new option does not break anything
+    check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
+    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
+    check_code(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
+    check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
 
 if __name__ == '__main__':
     main()

lnt/trunk/tests/server/ui/statsTester.py

+#
+# create temporary instance
+# Cleanup temporary directory in case one remained from a previous run - also
+# see PR9904.
+# RUN: rm -rf %t.instance
+# RUN: python %{shared_inputs}/create_temp_instance.py \
+# RUN:   %s %{shared_inputs}/SmallInstance %t.instance \
+# RUN:   %S/Inputs/V4Pages_extra_records.sql
+#
+# RUN: python %s %t.instance
+
+import unittest
+
+import lnt.util.stats as stats
+
+INDEX = 0
+
+
+class TestLNTStatsTester(unittest.TestCase):
+
+    def loc_test_agg_mean(self, values):
+        if values is None:
+            return stats.agg_mean(None)
+        agg_value, agg_index = stats.agg_mean((value, index) for (index, value) in enumerate(values))
+        return (agg_value, agg_index)
+
+    def test_agg_mean(self):
+        test_list1 = [1, 2, 3, 4, 6]
+        self.assertEqual(self.loc_test_agg_mean(test_list1), (3.2, INDEX))
+        test_list2 = [1.0, 2.0, 3.0, 4.0]
+        self.assertEqual(self.loc_test_agg_mean(test_list2), (2.5, INDEX))
+        test_list3 = [1.0]
+        self.assertEqual(self.loc_test_agg_mean(test_list3), (1.0, INDEX))
+        self.assertEqual(self.loc_test_agg_mean([]), (None, None))
+        self.assertEqual(self.loc_test_agg_mean(None), (None, None))
+
+        # Test it exactly how it is called in views.py without indirection
+        agg_value, agg_index = stats.agg_mean((value, index) for (index, value) in enumerate(test_list1))
+        self.assertEqual((3.2, INDEX), (agg_value, agg_index))
+        agg_value, agg_index = stats.agg_mean((value, index) for (index, value) in enumerate(test_list2))
+        self.assertEqual((2.5, INDEX), (agg_value, agg_index))
+        agg_value, agg_index = stats.agg_mean((value, index) for (index, value) in enumerate(test_list3))
+        self.assertEqual((1.0, INDEX), (agg_value, agg_index))
+
+if __name__ == '__main__':
+    try:
+        unittest.main()
+    except AttributeError:
+        # Command line parameters are treaded as test cases, when \
+        # running with lit rather than python directly.
+        import sys
+        if len(sys.argv) != 2:
+            sys.exit("Something went horribly wrong. You need parameters.")
+        del sys.argv[1:]
+        unittest.main()