[analyzer] [tests] Show detailed statistics comparision in CmpRuns

[mikhail.ramalho]

Summary

Introduces --stats-only option to only show changes in statistics.

[mikhail.ramalho]

Commandeering because of gsoc.

[mikhail.ramalho]

Fix statistics not being properly parsed

[NoQ]

Looks good!

utils/analyzer/CmpRuns.py

45–47

This probably won't work on Windows.
Not sure if we care.

293–296

Having a "total" statistic would be really useful, i think.

313–316

That's probably a matter of taste but i'd appreciate "after - before" here instead of "before - after". It's confusing that a positive value indicates a decrease in time, so it makes the reader keep more flips in the head. It won't be that confusing that a positive value indicates a non-positive outcome, because that's not part of the arithmetic. Ideally, have the "ratio" split out into a variable and get it compared to 0.2 and -0.2 respectively.

[mikhail.ramalho]

utils/analyzer/CmpRuns.py

45–47

Should I remove this?

[george.karpenkov]

utils/analyzer/CmpRuns.py

45–47

@NoQ what if we change if sys.stdout.isatty() to if sys.stdout.isatty() and not [isWindows] ?

[NoQ]

utils/analyzer/CmpRuns.py

45–47

Yeah, any approach will do :)

[mikhail.ramalho]

Addressed the following reviews:

• Added 'total' to the statistics
• Rewrote the ratio calculation

utils/analyzer/CmpRuns.py

313–316

Are you talking about how the calculation is being performed or how it's being printed on the screen?

[NoQ]

utils/analyzer/CmpRuns.py

313–316

How it's performed, i.e. B-A vs. A-B.

[mikhail.ramalho]

Rewrote the ratio calculation and added a comment saying that it doesn't highlight results in Windows.

[NoQ]

Thanks! Please commit.

[mikhail.ramalho]

Fix a division by zero when calculating the ratio

[george.karpenkov]

BTW this one should have been also uploaded with -U999 [or using arc]

[mikhail.ramalho]

Generate patch using -U999

[mikhail.ramalho]

Commited in r333529.

utils/analyzer/CmpRuns.py

Usage:
     resultsB = loadResultsFromSingleRun(singleRunInfoB, deleteEmpty)
 
     # Generate a relation from diagnostics in run A to diagnostics in run B
     # to obtain a list of triples (a, b, confidence).
     diff = compareResults(resultsA, resultsB)
 
 """
 
-import sys
-import os
-import plistlib
+from collections import defaultdict
+
 from math import log
 from optparse import OptionParser
+import json
+import os
+import plistlib
+import re
+import sys
 
+STATS_REGEXP = re.compile(r"Statistics: (\{.+\})", re.MULTILINE | re.DOTALL)
+
+class Colors:
+    """
+    Color for terminal highlight.
+    """
+    RED = '\x1b[2;30;41m'
+    GREEN = '\x1b[6;30;42m'
+    CLEAR = '\x1b[0m'
 
 # Information about analysis run:
 # path - the analysis output directory
 # root - the name of the root directory, which will be disregarded when
 # determining the source file name
 class SingleRunInfo:
     def __init__(self, path, root="", verboseLog=None):
         self.path = path
class AnalysisRun:
     def __init__(self, info):
         self.path = info.path
         self.root = info.root
         self.info = info
         self.reports = []
         # Cumulative list of all diagnostics from all the reports.
         self.diagnostics = []
         self.clang_version = None
+        self.stats = []
 
     def getClangVersion(self):
         return self.clang_version
 
     def readSingleFile(self, p, deleteEmpty):
         data = plistlib.readPlist(p)
+        if 'statistics' in data:
+            self.stats.append(json.loads(data['statistics']))
+            data.pop('statistics')
 
         # We want to retrieve the clang version even if there are no
         # reports. Assume that all reports were created using the same
         # clang version (this is always true and is more efficient).
         if 'clang_version' in data:
             if self.clang_version is None:
                 self.clang_version = data.pop('clang_version')
             else:
def compareResults(A, B, opts):
     if opts.relative_log_path_histogram or opts.relative_path_histogram or \
             opts.absolute_path_histogram:
         from matplotlib import pyplot
         pyplot.hist(path_difference_data, bins=100)
         pyplot.show()
 
     return res
 
+def deriveStats(results):
+    # Assume all keys are the same in each statistics bucket.
+    combined_data = defaultdict(list)
+    for stat in results.stats:
+        for key, value in stat.iteritems():
+            combined_data[key].append(value)
+    combined_stats = {}
+    for key, values in combined_data.iteritems():
+        combined_stats[str(key)] = {
+            "max": max(values),
+            "min": min(values),
+            "mean": sum(values) / len(values),
+            "median": sorted(values)[len(values) / 2],
+            "total": sum(values)
+        }
+    return combined_stats
+
+
+def compareStats(resultsA, resultsB):
+    statsA = deriveStats(resultsA)
+    statsB = deriveStats(resultsB)
+    keys = sorted(statsA.keys())
+    for key in keys:
+        print key
+        for kkey in statsA[key]:
+            valA = float(statsA[key][kkey])
+            valB = float(statsB[key][kkey])
+            report = "%.3f -> %.3f" % (valA, valB)
+            # Only apply highlighting when writing to TTY and it's not Windows
+            if sys.stdout.isatty() and os.name != 'nt':
+                if valB != 0:
+                  ratio = (valB - valA) / valB
+                  if ratio < -0.2:
+                      report = Colors.GREEN + report + Colors.CLEAR
+                  elif ratio > 0.2:
+                      report = Colors.RED + report + Colors.CLEAR
+            print "\t %s %s" % (kkey, report)
 
 def dumpScanBuildResultsDiff(dirA, dirB, opts, deleteEmpty=True,
                              Stdout=sys.stdout):
     # Load the run results.
     resultsA = loadResults(dirA, opts, opts.rootA, deleteEmpty)
     resultsB = loadResults(dirB, opts, opts.rootB, deleteEmpty)
+    if resultsA.stats:
+        compareStats(resultsA, resultsB)
+    if opts.stats_only:
+        return
 
     # Open the verbose log, if given.
     if opts.verboseLog:
         auxLog = open(opts.verboseLog, "wb")
     else:
         auxLog = None
 
     diff = compareResults(resultsA, resultsB, opts)
parser.add_option("--relative-log-path-differences-histogram",
                       default=False,
                       help="Show histogram of log relative paths differences. \
                             Requires matplotlib")
     parser.add_option("--absolute-path-differences-histogram",
                       action="store_true", dest="absolute_path_histogram",
                       default=False,
                       help="Show histogram of absolute paths differences. \
                             Requires matplotlib")
+    parser.add_option("--stats-only", action="store_true", dest="stats_only",
+                      default=False, help="Only show statistics on reports")
     return parser
 
 
 def main():
     parser = generate_option_parser()
     (opts, args) = parser.parse_args()
 
     if len(args) != 2: