[LNT] Add a "number of test results" table to the daily report page.

[kristof.beyls]

Summary

We've found it useful to have a table on the daily report page which reports for each
machine how many tests were seen on a given day. It has helped us to quickly notice
when a certain machine started producing fewer test/benchmark results for some reason.

AFAIK, there isn't another way to quickly notice if a particular machine starts producing
fewer-than-expected benchmark results.

[kristof.beyls]

Just adding a screenshot of what the table looks like on a day where the machines haven't produced a full set of benchmark results:

[cmatthews]

Looks good. Wonderful feature. Thanks!

lnt/trunk/lnt/server/reporting/analysis.py

class RunInfo(object):
 
     def get_run_comparison_result(self, run, compare_to, test_id, field):
         if compare_to is not None:
             compare_to = [compare_to]
         else:
             compare_to = []
         return self.get_comparison_result([run], compare_to, test_id, field)
 
-    def get_samples(self, runs, test_id, field):
+    def get_samples(self, runs, test_id):
         all_samples = []
         for run in runs:
             samples = self.sample_map.get((run.id, test_id))
             if samples is not None:
                 all_samples.extend(samples)
         return all_samples
 
     def get_comparison_result(self, runs, compare_runs, test_id, field):
         # Get the field which indicates the requested field's status.
         status_field = field.status_field
 
         # Load the sample data for the current and previous runs and the
         # comparison window.
-        run_samples = self.get_samples(runs, test_id, field)
-        prev_samples = self.get_samples(compare_runs, test_id, field)
+        run_samples = self.get_samples(runs, test_id)
+        prev_samples = self.get_samples(compare_runs, test_id)
 
         # Determine whether this (test,pset) passed or failed in the current and
         # previous runs.
         #
         # FIXME: Support XFAILs and non-determinism (mixed fail and pass)
         # better.
         run_failed = prev_failed = False
         if status_field:

lnt/trunk/lnt/server/reporting/dailyreport.py

def __init__(self, ts, year, month, day, num_prior_days_to_include=3,
         # Computed values.
         self.error = None
         self.next_day = None
         self.prior_days = None
         self.machine_runs = None
         self.reporting_machines = None
         self.reporting_tests = None
         self.result_table = None
+        self.nr_tests_table = None
 
     def get_query_parameters_string(self):
         query_params = [
             "{}={}".format(
                 urllib.quote_plus(query_param), urllib.quote_plus(str(value)))
             for query_param, value in (
                 ("day_start", self.day_start_offset.seconds / 3600),
                 ("num_days", self.num_prior_days_to_include),
def build(self):
 
                 if (test_status == REGRESSED or test_status == UNCHANGED_FAIL):
                     had_failures = True
                 elif day0_cr.pct_delta is not None:
                     sum_abs_day0_deltas += abs(day0_cr.pct_delta)
             return (-int(had_failures), -sum_abs_day0_deltas, test.name)
 
         self.result_table = []
+        self.nr_tests_table = []
         for field in self.fields:
             field_results = []
             for test in self.reporting_tests:
                 # For each machine, compute if there is anything to display for
                 # the most recent day, and if so add it to the view.
                 visible_results = []
                 for machine in self.reporting_machines:
                     # Get the most recent comparison result.
                     # Record which days have samples, so that we'll compare
                     # also consecutive runs that are further than a day
                     # apart if no runs happened in between.
                     day_has_samples = []
                     for i in range(0, self.num_prior_days_to_include):
                         runs = self.machine_past_runs.get((machine.id, i), ())
-                        samples = sri.get_samples(runs, test.id, field)
+                        samples = sri.get_samples(runs, test.id)
                         day_has_samples.append(len(samples) > 0)
 
                     def find_most_recent_run_with_samples(day_nr):
                         for i in range(day_nr+1,
                                        self.num_prior_days_to_include):
                             if day_has_samples[i]:
                                 return i
                         return day_nr+1
def build(self):
 
                 # If there are visible results for this test, append it to the
                 # view.
                 if visible_results:
                     field_results.append((test, visible_results))
 
             # Order the field results by "priority".
             field_results.sort(key=compute_visible_results_priority)
-
             self.result_table.append((field, field_results))
 
+        for machine in self.reporting_machines:
+            nr_tests_for_machine = []
+            for i in range(0, self.num_prior_days_to_include):
+                # get all runs with the same largest "order" on a given day
+                day_runs = machine_runs.get((machine.id, i), ())
+                nr_tests_seen = 0
+                for test in self.reporting_tests:
+                    samples = sri.get_samples(day_runs, test.id)
+                    if len(samples)>0:
+                        nr_tests_seen += 1
+                nr_tests_for_machine.append(nr_tests_seen)
+            self.nr_tests_table.append((machine, nr_tests_for_machine))
+
     def render(self, ts_url, only_html_body=True):
         # Strip any trailing slash on the testsuite URL.
         if ts_url.endswith('/'):
             ts_url = ts_url[:-1]
 
         env = lnt.server.ui.app.create_jinja_environment()
         template = env.get_template('reporting/daily_report.html')
 

lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html

<table border="1" style="{{ styles.table }}">
   <thead>
     <tr>
       <th style="{{ styles.th }}">Machine Name</th>
 {%- for i in range(report.num_prior_days_to_include)|reverse %}
       <th style="{{ styles.th }}">Day - {{i}}</th>
 {%- endfor %}
     </tr>
   </thead>
+  <tbody>
 {%- for machine in report.reporting_machines %}
     <tr class="searchable">
       <td class="machine-name" style="{{ styles.td }}">{{machine.name}}</td>
 {%- for i in range(report.num_prior_days_to_include)|reverse %}
 {%-   set key_run = report.get_key_run(machine, i) -%}
 {%-   if key_run -%}
       {# FIXME: Don't hard code field name. #}
       <td style="{{ styles.td }}">
         <a href="{{ ts_url }}/{{ key_run.id }}">{{
           key_run.order.llvm_project_revision}}</a></td>
 {%-   else -%}
       <td style="{{ styles.td }}; background-color:#FF0000">N/A</td>
 {%-   endif -%}
 {%- endfor %}
     </tr>
 {% endfor %}
+  </tbody>
 </table>
 
 {% macro get_initial_cell_value(day_result) %}
 {%- set cr = day_result.cr -%}
 {%- set test_status = cr.get_test_status() -%}
 
 {%- if (test_status == analysis.REGRESSED or
        test_status == analysis.UNCHANGED_FAIL) %}
<span>
           " fill="none" stroke="red" stroke-width="1"/>
         </g>
       </svg>
     </span>
 {%- endmacro %}
 
 {% for field,field_results in report.result_table|reverse %}
 <h3>Result Table ({{ field.name }})</h3>
-{{ result_header() if not report.for_mail }}
+{{ result_header()+"\n  <tbody>" if not report.for_mail }}
 {%- for test,visible_results in field_results -%}
-{{ result_header() if report.for_mail }}
+{{ result_header()+"\n  <tbody>" if report.for_mail }}
   <tr>
     <td style="{{ styles.td }}" colspan="2"> <b>{{test.name}}</b></td>
     <td style="{{ styles.td }}" colspan="{{report.num_prior_days_to_include + 1}}">&nbsp;</td>
   </tr>
 {%-   for machine,day_results in visible_results -%}
 {%-   set key_run = report.get_key_run(machine, 0) %}
   <tr class="searchable">
     <td style="{{ styles.td }}">&nbsp;</td>

 {%-        set first_result_shown = true -%}
     {{ get_initial_cell_value(day_result) }}
 {%-        endif -%}
 {%-      endif -%}
 {%-    endfor %}
     <td>{{ spark_plot(day_results) }}</td>
   </tr>
 {%-  endfor %}
-{{ "</table><p>" if report.for_mail }}
+{{ "</tbody></table><p>" if report.for_mail }}
 {%- endfor %}
-{{ "</table>" if not report.for_mail }}
+{{ "</tbody></table>" if not report.for_mail }}
 {%- endfor -%}
 
+<h3>Number of Tests Seen</h3>
+<table border="1" style="{{ styles.table }}; width: {{
+                            400 + report.num_prior_days_to_include*80}}px;">
+  <thead>
+    <tr>
+      <th style="{{ styles.th }}; width: 300px;">Machine Name</th>
+    {% for i in range(report.num_prior_days_to_include)|reverse %}
+      <th style="{{ styles.th }}">Day - {{i}}</th>
+    {% endfor %}
+    </tr>
+  </thead>
+  <tbody>
+  {% for machine, nr_tests_for_machine in report.nr_tests_table %}
+  <tr class="searchable">
+    <td class="machine-name" style="{{ styles.td }}">{{machine.name}}</td>
+    {% set nr_tests_prev = None %}
+    {% for nr_tests in nr_tests_for_machine|reverse %}
+        <td style="{{ styles.td }}"
+        {%- if nr_tests_prev is not none and nr_tests_prev != nr_tests %}
+            bgcolor="#dfaf5f"
+        {% endif -%}
+            >{{nr_tests}}</td>
+        {% set nr_tests_prev = nr_tests %}
+    {% endfor %}
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
+
 {% endif %}
 
 {% if not only_html_body %}
 </body>
 </html>
 {% endif %}

lnt/trunk/tests/server/ui/V4Pages.py

def check_nr_machines_reported(client, url, expected_nr_machines):
     # do this by looking for the title containing "Reported Machine Order"
     # and assuming that the first <table> at the same level after it is the
     # one we're looking for.
     reported_machine_order_table = \
         find_table_with_heading(tree, 'Reported Machine Order')
     if reported_machine_order_table is None:
         nr_machines = 0
     else:
-        nr_machines = len(reported_machine_order_table.findall("./tr"))
+        nr_machines = len(reported_machine_order_table.findall("./tbody/tr"))
     assert expected_nr_machines == nr_machines
 
 
 def convert_html_to_text(element):
     return ("".join(element.itertext()))
 
 
-def get_results_table(client, url, fieldname):
+def get_table_by_header(client, url, table_header):
     resp = check_code(client, url)
     html = resp.data
     tree = get_xml_tree(html)
-    table_header = "Result Table (%s)" % fieldname
     table = find_table_with_heading(tree, table_header)
     assert table is not None, \
         "Couldn't find table with header '%s'" % table_header
     return table
 
 
-def check_body_result_table(client, url, fieldname,
-                            expected_table_body_content):
-    table = get_results_table(client, url, fieldname)
+def get_results_table(client, url, fieldname):
+    table_header = "Result Table (%s)" % fieldname
+    return get_table_by_header(client, url, table_header)
+
+
+def check_table_content(table, expected_content):
     body_content = [[convert_html_to_text(cell).strip()
                      for cell in row.findall("./td")]
-                    for row in table.findall("./tr")]
-    assert expected_table_body_content == body_content, \
+                    for row in table.findall("./tbody/tr")]
+    assert expected_content == body_content, \
         "Expected table content %s, found %s" % \
-        (expected_table_body_content, body_content)
+        (expected_content, body_content)
+
+
+def check_body_result_table(client, url, fieldname, expected_content):
+    table = get_results_table(client, url, fieldname)
+    check_table_content(table, expected_content)
+
+
+def check_body_nr_tests_table(client, url, expected_content):
+    table_header = "Number of Tests Seen"
+    table = get_table_by_header(client, url, table_header)
+    check_table_content(table, expected_content)
 
 
 def get_sparkline(client, url, fieldname, testname, machinename):
     table = get_results_table(client, url, fieldname)
     body_content = [[cell
                      for cell in row.findall("./td")]
-                    for row in table.findall("./tr")]
+                    for row in table.findall("./tbody/tr")]
     txt_body_content = [[convert_html_to_text(cell).strip()
                          for cell in row.findall("./td")]
-                        for row in table.findall("./tr")]
+                        for row in table.findall("./tbody/tr")]
     cur_test_name = ""
     for rownr, row_content in enumerate(txt_body_content):
-        nr_columns = len(row_content)
         for colnr, col_content in enumerate(row_content):
             if colnr == 0 and col_content != "":
                 cur_test_name = col_content
             if colnr == 1 and col_content != "":
                 cur_machine_name = machinename
                 if (cur_machine_name, cur_test_name) == \
                    (machinename, testname):
                     return body_content[rownr][-1]
check_body_result_table(client,
                              ["", "machine2", "1.0000", "FAIL", "PASS", ""]])
     sparkline_xml = get_sparkline(client,
                                   '/v4/nts/daily_report/2012/5/13?num_days=3',
                                   "execution_time", "test6", "machine2")
     nr_sample_points = len(extract_sample_points(sparkline_xml))
     assert 2 == nr_sample_points, \
         "Expected 2 sample points, found %d" % nr_sample_points
 
+    check_body_nr_tests_table(
+        client, '/v4/nts/daily_report/2012/5/04',
+        [['machine2', '2', '0', '1']])
+
+
+
+
     # Now check the compile report
     # Get the V4 overview page.
     check_code(client, '/v4/compile/')
 
     # Get a machine overview page.
     check_code(client, '/v4/compile/machine/1')
     check_code(client, '/v4/compile/machine/2')
     # Don't crash when requesting non-existing data