Color the background of sparklines on the daily report page based on the hash value of the binary.

[kristof.beyls]

Summary

This depends on http://reviews.llvm.org/D16355.

[kristof.beyls]

Adding screenshot of what the background colouring in sparklines based on binary hash values looks like. The screenshot shows that it's more straightforward to see what results are noise and which ones are probably real differences:

• The 14% performance swing in viterbi in the screenshot is just noise, as the program binary didn't change (same background colour).
• The 13% performance swing in sim is probably real: the binary changed since yesterday.
• hexxagon and sqlite3: the regressions are probably real. Both programs have oscillated a bit over the past week; i.e. the re-occurence of the same background colour indicates LLVM ToT's codegen for these programs has been identical on a few non-consecutive days.
• hash2: it's just noise, the binary hasn't changed for the past 10 days.

[cmatthews]

Mostly looks good. Could you move the color selection logic out of analysis.py and into lnt/server/ui/util.py? Analysis seems like a strange place for color selection logic to live.

[MatzeB]

Isn't the only useful information here whether the binary has changed compared to the previous revision? Wouldn't it be better to just check for that and only alternate between two colors?

[MatzeB]

In D16356#331807, @MatzeB wrote:

Isn't the only useful information here whether the binary has changed compared to the previous revision? Wouldn't it be better to just check for that and only alternate between two colors?

I just realized that you are exactly reusing a color (within a sparcline) iff the checksum is the same as a previous one, that makes sense and conveys extra information. So please ignore my previous comment :)

[kristof.beyls]

Ah yes, indeed the color selection logic fits much better in ui/util.py rather than analysis.py. Thanks for the pointer!
This updated patch has moved it to util.py and fixed the pep8 violations present in the original patch.

[cmatthews]

LGTM

[kristof.beyls]

In D16356#332484, @cmatthews wrote:

LGTM

Many thanks for the review!
Would you be able to also have a quick look at http://reviews.llvm.org/D16355, as this patch depends on that one?

Thanks!

Kristof

lnt/trunk/lnt/server/reporting/dailyreport.py

def complete(self):
         for dr in self.day_results:
             if dr is None:
                 continue
             if dr.cr.samples is not None and not dr.cr.failed:
                 all_samples.extend(dr.cr.samples)
         if len(all_samples) > 0:
             self.min_sample = min(all_samples)
             self.max_sample = max(all_samples)
+        hashes = []
+        for dr in self.day_results:
+            if dr is None:
+                hashes.append(None)
+            else:
+                hashes.append(dr.hash)
+        rgb_colors = util.get_rgb_colors_for_hashes(hashes)
+        for i, dr in enumerate(self.day_results):
+            if dr is not None:
+                dr.hash_rgb_color = rgb_colors[i]
 
 
 class DailyReport(object):
     def __init__(self, ts, year, month, day, num_prior_days_to_include=3,
                  day_start_offset_hours=16, for_mail=False,
                  filter_machine_regex=None):
         self.ts = ts
         self.hash_of_binary_field = self.ts.Sample.get_hash_of_binary_field()

lnt/trunk/lnt/server/ui/templates/reporting/daily_report.html

<table border="1" style="{{ styles.table }}; width: {{
   {%- set scaling_factor = 1.0 -%}
 {%- endif %}
 {%- macro spark_y_coord(day_nr, value) -%}
   {{ (value-day_results.min_sample) * scaling_factor + y_border_size }}
 {%- endmacro -%}
 {%- macro spark_x_coord(day_nr) -%}
   {{ (nr_days - day_nr) * x_day_spacing + x_border_size }}
 {%- endmacro -%}
+{%- macro spark_hash_background(day_nr, dr) -%}
+  {%- if dr.cr.cur_hash is not none -%}
+    {%- set style = "fill: "+dr.hash_rgb_color+";" -%}
+  {%- else -%}
+    {%- set style = "fill: none;" -%}
+  {%- endif -%}
+    <rect x="{{(nr_days-day_nr-0.5) * x_day_spacing + x_border_size}}" y="0"
+          width="{{x_day_spacing}}" height="{{full_height}}" style="{{style}}"/>
+{%- endmacro -%}
     <span>
       <svg width="{{width}}" height="{{full_height}}">
-      <rect width="{{width}}" height="{{full_height}}" fill="beige"/>
+      <rect width="{{width}}" height="{{full_height}}" fill="#f7f7f7"/>
 {#- Make y-axis go upwards instead of downwards: #}
       <g transform="translate(0, {{full_height}}) scale(1, -1) ">
 {%- for dr in day_results -%}
   {%- if dr is not none and not dr.cr.failed -%}
     {%- set day_nr = loop.index %}
     {%- set nr_samples_for_day = dr.samples|length %}
+        {{ spark_hash_background(day_nr, dr) }}
     {%- for sample in dr.samples -%}
       {# fuzz the x-coordinate slightly so that multiple samples with the same
          value can be noticed #}
       {%- set sample_fuzz = (-sample_fuzzing*1.25) +
                        (2.0*sample_fuzzing/nr_samples_for_day) * loop.index %}
         <circle cx="{{ spark_x_coord(day_nr)|float + sample_fuzz }}" {# -#}
                 cy="{{ spark_y_coord(day_nr, sample) }}" r="1"
                 stroke-width="1" stroke="black" fill="black" />

lnt/trunk/lnt/server/ui/util.py

def makeLightColor(h):
     return colorsys.hsv_to_rgb(h,s,v)
 
 def makeBetterColor(h):
     h = math.cos(h*math.pi*.5)
     s = .8 + ((math.cos(h * math.pi*.5) + 1)*.5) * .2
     v = .88
     return colorsys.hsv_to_rgb(h,s,v)
 
+
+# The hash color palette avoids green and red as these colours are already used
+# in quite a few places to indicate "good" or "bad".
+hash_color_palette = (
+    colorsys.hsv_to_rgb(h=45./360,  s=0.3, v=0.9999),  # warm yellow
+    colorsys.hsv_to_rgb(h=210./360, s=0.3, v=0.9999),  # blue cyan
+    colorsys.hsv_to_rgb(h=300./360, s=0.3, v=0.9999),  # mid magenta
+    colorsys.hsv_to_rgb(h=150./360, s=0.3, v=0.9999),  # green cyan
+    colorsys.hsv_to_rgb(h=225./360, s=0.3, v=0.9999),  # cool blue
+    colorsys.hsv_to_rgb(h=180./360, s=0.3, v=0.9999),  # mid cyan
+)
+
+
+def get_rgb_colors_for_hashes(hash_strings):
+    hash2color = {}
+    unique_hash_counter = 0
+    for hash_string in hash_strings:
+        if hash_string is not None:
+            if hash_string in hash2color:
+                continue
+            hash2color[hash_string] = hash_color_palette[unique_hash_counter]
+            unique_hash_counter += 1
+            if unique_hash_counter >= len(hash_color_palette):
+                break
+    result = []
+    for hash_string in hash_strings:
+        if hash_string is None:
+            result.append(None)
+        else:
+            # If not one of the first N hashes, return rgb value 0,0,0 which is
+            # white.
+            rgb = hash2color.get(hash_string, (0.999, 0.999, 0.999))
+            result.append(toColorString(rgb))
+    return result
+
+
 class multidict:
     def __init__(self, elts=()):
         self.data = {}
         for key,value in elts:
             self[key] = value
 
     def __contains__(self, item):
         return item in self.data

lnt/trunk/tests/SharedInputs/SmallInstance/data/lnt_db_create.sql

 BEGIN TRANSACTION;
 CREATE TABLE "SchemaVersion" (
   "Name" VARCHAR(256) NOT NULL, 
   "Version" INTEGER, 
   PRIMARY KEY ("Name"), 
   UNIQUE ("Name")
 );
-INSERT INTO "SchemaVersion" VALUES('__core__',6);
+INSERT INTO "SchemaVersion" VALUES('__core__',7);
 CREATE TABLE "TestSuite" (
   "ID" INTEGER PRIMARY KEY NOT NULL, 
   "Name" VARCHAR(256), 
   "DBKeyName" VARCHAR(256), 
   "Version" VARCHAR(16), 
   UNIQUE ("Name")
 );
 INSERT INTO "TestSuite" ("Name", "DBKeyName") VALUES('nts','NT');         --ID 1

 INSERT INTO "StatusKind" VALUES(2,'XFAIL');
 CREATE TABLE "SampleType" (
   "ID" INTEGER PRIMARY KEY NOT NULL, 
   "Name" VARCHAR(256), 
   UNIQUE ("Name")
 );
 INSERT INTO "SampleType" ("Name") VALUES('Real');   -- ID 1
 INSERT INTO "SampleType" ("Name") VALUES('Status'); -- ID 2
+INSERT INTO "SampleType" ("Name") VALUES('Hash');   -- ID 3
 CREATE TABLE "TestSuiteRunFields" (
   "ID" INTEGER PRIMARY KEY NOT NULL, 
   "TestSuiteID" INTEGER, 
   "Name" VARCHAR(256), 
   "InfoKey" VARCHAR(256), 
   FOREIGN KEY("TestSuiteID") REFERENCES "TestSuite" ("ID")
 );
 CREATE TABLE "TestSuiteOrderFields" (
INSERT INTO "TestSuiteSampleFields" ("TestSuiteID", "Name", "Type", "InfoKey",
                                      "status_field", "bigger_is_better")
  VALUES(2,'mem_bytes',1,'.mem',9,0);                    -- ID 14
 INSERT INTO "TestSuiteSampleFields" ("TestSuiteID", "Name", "Type", "InfoKey",
                                      "status_field", "bigger_is_better")
  VALUES(1,'score',1,'.score',NULL,1);                   -- ID 15
 INSERT INTO "TestSuiteSampleFields" ("TestSuiteID", "Name", "Type", "InfoKey",
                                      "status_field", "bigger_is_better")
  VALUES(1,'mem_bytes',1,'.mem',NULL,0);                 -- ID 16
+INSERT INTO "TestSuiteSampleFields" ("TestSuiteID", "Name", "Type", "InfoKey",
+                                     "status_field", "bigger_is_better")
+ VALUES(1,'hash_status',2,'.hash.status',NULL,0);       -- ID 17
+INSERT INTO "TestSuiteSampleFields" ("TestSuiteID", "Name", "Type", "InfoKey",
+                                     "status_field", "bigger_is_better")
+ VALUES(1,'hash',3,'.hash',NULL,0);                     -- ID 18
 CREATE TABLE "TestSuiteMachineFields" (
   "ID" INTEGER PRIMARY KEY NOT NULL, 
   "TestSuiteID" INTEGER, 
   "Name" VARCHAR(256), 
   "InfoKey" VARCHAR(256), 
   FOREIGN KEY("TestSuiteID") REFERENCES "TestSuite" ("ID")
 );
 INSERT INTO "TestSuiteMachineFields" ("TestSuiteID", "Name", "InfoKey")

 CREATE TABLE "NT_Sample" (
   "ID" INTEGER PRIMARY KEY NOT NULL, 
   "RunID" INTEGER, 
   "TestID" INTEGER, 
   compile_status INTEGER, 
   execution_status INTEGER, 
   compile_time FLOAT, 
   execution_time FLOAT, score FLOAT, "mem_bytes" FLOAT, 
+        hash_status INTEGER, hash VARCHAR(32),
   FOREIGN KEY("RunID") REFERENCES "NT_Run" ("ID"), 
   FOREIGN KEY("TestID") REFERENCES "NT_Test" ("ID"), 
   FOREIGN KEY(compile_status) REFERENCES "StatusKind" ("ID"), 
-  FOREIGN KEY(execution_status) REFERENCES "StatusKind" ("ID")
+  FOREIGN KEY(execution_status) REFERENCES "StatusKind" ("ID"),
+  FOREIGN KEY(hash_status) REFERENCES "StatusKind" ("ID")
 );
 INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
                          "execution_status", "compile_time", "execution_time",
                          "score", "mem_bytes")
  VALUES(1,1,NULL,NULL,0.007,0.0003,NULL,NULL); -- ID 1
 INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
                          "execution_status", "compile_time", "execution_time",
                          "score", "mem_bytes")

lnt/trunk/tests/server/ui/Inputs/V4Pages_extra_records.sql

INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
                          "score", "mem_bytes")
  VALUES(8,6,0,1,0.001,1.0,NULL,NULL); -- ID 10: failing result
 INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
                          "execution_status", "compile_time", "execution_time",
                          "score", "mem_bytes")
  VALUES(9,6,0,0,0.001,1.2,NULL,NULL); -- ID 11: passing result; 20% bigger,
                                       -- so shown in daily report page.
 
+-- check background colors being produced correctly, corresponding to recorded
+-- hashes of the binary.
+INSERT INTO "NT_Test" VALUES(7,'test_hash1'); -- ID 7
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(7,7,0,0,0.001,1.0,NULL,NULL,0,'hash1'); -- ID 11: hash1
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(8,7,0,0,0.001,1.0,NULL,NULL,NULL,NULL); -- ID 12: no hash
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(9,7,0,0,0.001,1.2,NULL,NULL,0,'hash2'); -- ID 13: hash2; 20% bigger,
+                                      -- so shown in daily report page.
+
+INSERT INTO "NT_Test" VALUES(8,'test_hash2'); -- ID 8
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(7,8,0,0,0.001,1.0,NULL,NULL,0,'hash1'); -- ID 14: hash1
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(8,8,0,0,0.001,1.0,NULL,NULL,0,'hash2'); -- ID 15: hash2
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(9,8,0,0,0.001,1.2,NULL,NULL,0,'hash1'); -- ID 16: hash1; 20% bigger,
+                                      -- so shown in daily report page.
+
+INSERT INTO "NT_Test" VALUES(9,'test_mhash_on_run'); -- ID 9
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(7,9,0,0,0.001,1.0,NULL,NULL,0,'hash1'); -- ID 15: hash1
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(7,9,0,0,0.001,1.0,NULL,NULL,0,'hash2'); -- ID 16: hash2, same day
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(8,9,0,0,0.001,1.0,NULL,NULL,1,NULL); -- ID 17: no hash the next day
+INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
+                         "execution_status", "compile_time", "execution_time",
+                         "score", "mem_bytes", "hash_status", "hash")
+ VALUES(9,9,0,0,0.001,1.2,NULL,NULL,0,'hash3'); -- ID 18: hash3; 20% bigger,
+                                      -- so shown in daily report page.
+
 
 COMMIT;

lnt/trunk/tests/server/ui/V4Pages.py

 
 
 def check_body_nr_tests_table(client, url, expected_content):
     table_header = "Number of Tests Seen"
     table = get_table_by_header(client, url, table_header)
     check_table_content(table, expected_content)
 
 
-def get_sparkline(client, url, fieldname, testname, machinename):
-    table = get_results_table(client, url, fieldname)
+def get_sparkline(table, testname, machinename):
     body_content = [[cell
                      for cell in row.findall("./td")]
                     for row in table.findall("./tbody/tr")]
     txt_body_content = [[convert_html_to_text(cell).strip()
                          for cell in row.findall("./td")]
                         for row in table.findall("./tbody/tr")]
     cur_test_name = ""
     for rownr, row_content in enumerate(txt_body_content):

 
 
 def extract_sample_points(sparkline_svg):
     # assume all svg:circle elements are exactly all the sample points
     samples = sparkline_svg.findall(".//circle")
     return samples
 
 
+fillStyleRegex = re.compile("fill: *(?P<fill>[^;]+);")
+
+
+def extract_background_colors(sparkline_svg, nr_days):
+    rects = sparkline_svg.findall(".//rect")
+    # The first rectangle returned is the default background, so remove that
+    # one.
+    assert len(rects) >= 1
+    rects = rects[1:]
+    result = []
+    for rect in rects:
+        style = rect.get("style", None)
+        if style is None:
+            result.append(None)
+            continue
+        m = fillStyleRegex.search(style)
+        if m is None:
+            result.append(None)
+            continue
+        fill = m.group('fill')
+        if fill == 'none':
+            result.append(None)
+        else:
+            result.append(fill)
+    return result
+
+
 def main():
     _, instance_path = sys.argv
 
     # Create the application instance.
     app = lnt.server.ui.app.App.create_standalone(instance_path)
 
     # Don't catch out exceptions.
     app.testing = True
check_body_result_table(client, '/v4/nts/daily_report/2012/5/04',
                              ["", "machine2", "FAIL", "-", "PASS", ""]])
 
     # Check that a failing result does not show up in the spark line
     # as a dot with value 0.
     check_body_result_table(client,
                             '/v4/nts/daily_report/2012/5/13?num_days=3',
                             "execution_time",
                             [["test6", ""],
-                             ["", "machine2", "1.0000", "FAIL", "PASS", ""]])
-    sparkline_xml = get_sparkline(client,
-                                  '/v4/nts/daily_report/2012/5/13?num_days=3',
-                                  "execution_time", "test6", "machine2")
-    nr_sample_points = len(extract_sample_points(sparkline_xml))
+                             ["", "machine2", "1.0000", "FAIL", "PASS", ""],
+                             ["test_hash1", ""],
+                             ["", "machine2", "1.0000", '-', '20.00%', ""],
+                             ["test_hash2", ""],
+                             ["", "machine2", "1.0000", '-', '20.00%', ""],
+                             ["test_mhash_on_run", ""],
+                             ["", "machine2", "1.0000", '-', '20.00%', ""], ])
+    result_table = get_results_table(
+        client, '/v4/nts/daily_report/2012/5/13?num_days=3', "execution_time")
+    sparkline_test6_xml = get_sparkline(result_table, "test6", "machine2")
+    nr_sample_points = len(extract_sample_points(sparkline_test6_xml))
     assert 2 == nr_sample_points, \
         "Expected 2 sample points, found %d" % nr_sample_points
 
     check_body_nr_tests_table(
         client, '/v4/nts/daily_report/2012/5/04',
         [['machine2', '2', '0', '1']])
 
-
-
+    # Check that a different background color is used in the sparkline
+    # when the hash values recorded are different. At the same time,
+    # check that no background color is drawn on missing hash values,
+    # using a sequence of (hash1, no hash, hash2) over 3 consecutive
+    # days.
+    sparkline_hash1_xml = get_sparkline(result_table, "test_hash1", "machine2")
+    nr_sample_points = len(extract_sample_points(sparkline_hash1_xml))
+    assert 3 == nr_sample_points, \
+        "Expected 3 sample points, found %d" % nr_sample_points
+    background_colors = extract_background_colors(sparkline_hash1_xml, 3)
+    assert len(background_colors) == 3
+    color1, color2, color3 = background_colors
+    assert color1 is not None
+    assert color3 is not None
+    assert color1 != color3
+    assert color2 is None
+
+    # Check that the same background color is used in the sparkline
+    # when the hash values recorded are the same, using a
+    # (hash1, hash2, hash1) sequence.
+    sparkline_hash2_xml = get_sparkline(result_table, "test_hash2", "machine2")
+    nr_sample_points = len(extract_sample_points(sparkline_hash2_xml))
+    assert 3 == nr_sample_points, \
+        "Expected 3 sample points, found %d" % nr_sample_points
+    background_colors = extract_background_colors(sparkline_hash2_xml, 3)
+    assert len(background_colors) == 3
+    color1, color2, color3 = background_colors
+    assert color1 is not None
+    assert color1 == color3
+    assert color1 != color2
+    assert color2 is not None
+
+    # Check that we don't crash if a single run produces multiple
+    # samples with different hash values for the same run. This could
+    # happen e.g. when the compiler under test doesn't produce
+    # object code deterministically.
+    sparkline_mhashonrun_xml = get_sparkline(
+        result_table, "test_mhash_on_run", "machine2")
+    nr_sample_points = len(extract_sample_points(sparkline_mhashonrun_xml))
+    assert 4 == nr_sample_points, \
+        "Expected 4 sample points, found %d" % nr_sample_points
+    background_colors = extract_background_colors(sparkline_mhashonrun_xml, 3)
+    assert len(background_colors) == 3
+    color1, color2, color3 = background_colors
+    assert color2 is None
+    assert color1 != color3
 
     # Now check the compile report
     # Get the V4 overview page.
     check_code(client, '/v4/compile/')
 
     # Get a machine overview page.
     check_code(client, '/v4/compile/machine/1')
     check_code(client, '/v4/compile/machine/2')