[utils] Add script to check for code coverage regressions

[vsk]

Summary

This script is meant to be run on code coverage bots in conjunction with utils/prepare-code-coverage-artifact.py to flag coverage regressions in a window of commits.

[vsk]

• Added doctests.
• Improved comments.
• Fixed an issue where we would exit(0) even if there are regressions.

[silvas]

lgtm, seems handy.

Is this something that you intend to ship to users, or just a tool for tracking LLVM coverage?

[silvas]

lgtm, seems handy.

Is this something that you intend to ship to users, or just a tool for tracking LLVM coverage?

[vsk]

In D22544#490472, @silvas wrote:

lgtm, seems handy.

Is this something that you intend to ship to users, or just a tool for tracking LLVM coverage?

Thanks. @cmatthews and I are working on a bot to track coverage regressions in llvm. The plan is to let people hammer out usability issues with internal bots (at least, that's what we're doing) and then see if there's broader interest.

I don't know what shipping this script to users entails. It doesn't bake in assumptions about the llvm codebase, if that's what you mean.

utils/check-coverage-regressions.py

+#!/usr/bin/env python
+
+'''Compare two coverage summaries for regressions.
+
+You can create a coverage summary by using the `llvm-cov report` command.
+Alternatively, you can use `utils/prepare-code-coverage-artifact.py` which
+creates summaries as well as file-based html reports.
+'''
+
+from __future__ import print_function
+
+import argparse
+import collections
+import re
+import sys
+
+# This threshold must be in [0, 1]. The lower the threshold, the less likely
+# it is that a regression will be flagged and vice versa.
+kThresh = 1.0
+
+FileCoverage = collections.namedtuple('FileCoverage',
+        ['Regions', 'Missed', 'Coverage', 'Functions', 'Executed'])
+
+CoverageEntry = re.compile('^(.*) +(\d+) +(\d+) +([\d.]+)% +(\d+) +([\d.]+)%$')
+
+def parse_file_coverage_line(line):
+    '''Parse @line as a summary of a file's coverage information.
+
+    >>> parse_file_coverage_line('foo.cpp 10 5 50.0% 10 0.0%')
+    ('foo.cpp', FileCoverage(Regions=10, Missed=5, Coverage=50.0, Functions=10, Executed=0.0))
+    '''
+
+    m = re.match(CoverageEntry, line)
+    if not m:
+        print("Could not read coverage summary:", line)
+        exit(1)
+
+    groups = m.groups()
+    filename = groups[0].strip()
+    regions = int(groups[1])
+    missed = int(groups[2])
+    coverage = float(groups[3])
+    functions = int(groups[4])
+    executed = float(groups[5])
+    return (filename,
+            FileCoverage(regions, missed, coverage, functions, executed))
+
+def parse_summary(path):
+    '''Parse the summary at @path. Return a dictionary mapping filenames to
+       FileCoverage instances.'''
+
+    with open(path, 'r') as f:
+        lines = f.readlines()
+
+    # Drop the header and the cell dividers. Include "TOTAL" in this list.
+    file_coverages = lines[2:-2] + [lines[-1]]
+
+    summary = {}
+    for line in file_coverages:
+        filename, fc = parse_file_coverage_line(line)
+        summary[filename] = fc
+    return summary
+
+def find_coverage_regressions(old_coverage, new_coverage):
+    '''Given two coverage summaries, generate coverage regressions of the form:
+       (filename, old FileCoverage, new FileCoverage).'''
+
+    for filename in old_coverage.keys():
+        if filename not in new_coverage:
+            continue
+
+        old_fc = old_coverage[filename]
+        new_fc = new_coverage[filename]
+        if new_fc.Coverage < kThresh * old_fc.Coverage or \
+                new_fc.Executed < kThresh * old_fc.Executed:
+            yield (filename, old_fc, new_fc)
+
+def print_regression(filename, old_fc, new_fc):
+    '''Pretty-print a coverage regression in @filename. @old_fc is the old
+       FileCoverage and @new_fc is the new one.
+
+    >>> print_regression('foo.cpp', FileCoverage(10, 5, 50.0, 10, 0), \
+                         FileCoverage(10, 7, 30.0, 10, 0))
+    Code coverage regression:
+      File: foo.cpp
+      Change in region coverage: -20.00%
+      Change in function coverage: 0.00%
+      No functions were added or removed.
+      No regions were added or removed.
+
+    >>> print_regression('foo.cpp', FileCoverage(10, 5, 50.0, 10, 0), \
+                         FileCoverage(5, 4, 20.0, 5, 0))
+    Code coverage regression:
+      File: foo.cpp
+      Change in region coverage: -30.00%
+      Change in function coverage: 0.00%
+      Change in the number of functions: -5
+      Change in the number of regions: -5
+    '''
+
+    region_coverage_delta = new_fc.Coverage - old_fc.Coverage
+    func_coverage_delta = new_fc.Executed - old_fc.Executed
+    num_functions_delta = new_fc.Functions - old_fc.Functions
+    num_regions_delta = new_fc.Regions - old_fc.Regions
+    print("Code coverage regression:")
+    print("  File:", filename)
+    print("  Change in region coverage: {0:.2f}%".format(region_coverage_delta))
+    print("  Change in function coverage: {0:.2f}%".format(func_coverage_delta))
+    if num_functions_delta:
+        print("  Change in the number of functions:", num_functions_delta)
+    else:
+        print("  No functions were added or removed.")
+    if num_regions_delta:
+        print("  Change in the number of regions:", num_regions_delta)
+    else:
+        print("  No regions were added or removed.")
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('old_summary', help='Path to the old coverage summary')
+    parser.add_argument('new_summary', help='Path to the new coverage summary')
+    args = parser.parse_args()
+
+    old_coverage = parse_summary(args.old_summary)
+    new_coverage = parse_summary(args.new_summary)
+
+    num_regressions = 0
+    for filename, old_fc, new_fc in \
+            find_coverage_regressions(old_coverage, new_coverage):
+        print_regression(filename, old_fc, new_fc)
+        num_regressions += 1
+
+    if num_regressions > 0:
+        exit(1)
+    exit(0)