LLVM Test suite Python 2/3 compatibility

[michaelplatings]

From looking briefly it seems that these are sensible changes, but many of them aren't necessary for Python 3 compatility. I suggest separating the Python 3 compatibility changes and other cleanups into different patches.

[serge-sans-paille]

@michaelplatings : some of the non trivial changes are just here to make it easier to write the compatible code. e.g.

for f in range(len(dbg_line_items)):
    fname = dbg_line_items[f][0]
    fset = dbg_line_items[f][1]

forces dbg_line_items to be a list, while

for fname, fset in dbg_line_items:

is compatible with both list and generators

[serge-sans-paille]

@michaelplatings I've split this patch in two so that it's easier to review, see https://reviews.llvm.org/D55989 for the companion patch.

[michaelplatings]

LGTM

test-suite/trunk/CompareDebugInfo.py

 #!/usr/bin/python
+from __future__ import print_function
 
 import os
 import sys
 
 DBG_OUTPUT_FILE="Output/" + sys.argv[1] + ".dbg.out"
 OPT_DBG_OUTPUT_FILE="Output/" + sys.argv[1] + ".dbg.opt.out"
 LOG_FILE="Output/" + sys.argv[1] + ".log"
 NATIVE_DBG_OUTPUT_FILE="Output/" + sys.argv[1] + ".native.dbg.out"
def getNotMatchingArgCount(self):
         if self.missing_bp == True:
             return 0
         return len(self.notmatching_args)
 
     def recordArgument(self, arg_name, value):
         self.values[arg_name] = value
         
     def __repr__(self):
-        print self.name
+        print(self.name)
         for k, v in self.values.items():
-            print k, "=", v
+            print(k, "=", v)
         return ''
 
     def compare_args(self, other, file):
         myitems = self.values.items()
-        otheritems = other.values.items()
+        otheritems = list(other.values.items())
         match = False
         for i, my_item in enumerate(my_items):
             if i >= len(otheritems):
                 match = True
                 self.missing_args.append(myitem[0])
             elif cmp(myitem[1], otheritems[i][1]):
                 match = True
                 self.notmatching_args.append(myitem[0])

test-suite/trunk/litsupport/modules/stats.py

 """test-suite/lit plugin to collect internal llvm json statistics.
 
 This assumes the benchmarks were built with the -save-stats=obj flag."""
 import json
 import logging
 import os
 from collections import defaultdict
 
 
 def _mergeStats(global_stats, statsfilename):
     try:
         f = open(statsfilename, "rt")
         stats = json.load(f)
     except Exception as e:
         logging.warning("Could not read '%s'", statsfilename, exc_info=e)
         return
-    for name, value in stats.iteritems():
+    for name, value in stats.items():
         global_stats[name] += value
 
 
 def _getStats(context):
     # We compile multiple benchmarks in the same directory in SingleSource
     # mode. Only look at compiletime files starting with the name of our test.
     prefix = ""
     if context.config.single_source:
         prefix = "%s." % os.path.basename(context.executable)
 
     stats = defaultdict(lambda: 0.0)
     dir = os.path.dirname(context.test.getFilePath())
     for path, subdirs, files in os.walk(dir):
         for file in files:
             if file.endswith('.stats') and file.startswith(prefix):
                 fullpath = os.path.join(path, file)
                 _mergeStats(stats, fullpath)
 
     if len(stats) == 0:
         logging.warning("No stats for '%s'", context.test.getFullName())
 
     result = dict()
-    for key, value in stats.iteritems():
+    for key, value in stats.items():
         result[key] = value
     return result
 
 
 def mutatePlan(context, plan):
     plan.metric_collectors.append(_getStats)

test-suite/trunk/utils/compare.py

-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 """Tool to filter, organize, compare and display benchmarking results. Usefull
 for smaller datasets. It works great with a few dozen runs it is not designed to
 deal with hundreds.
 Requires the pandas library to be installed."""
+from __future__ import print_function
+
 import pandas as pd
 import sys
 import os.path
 import re
 import numbers
 import argparse
 
 def read_lit_json(filename):
     import json
     jsondata = json.load(open(filename))
     columns = []
     columnindexes = {}
     names = set()
     info_columns = ['hash']
     # Pass1: Figure out metrics (= the column index)
     if 'tests' not in jsondata:
-        print "%s: Could not find toplevel 'tests' key"
+        print("%s: Could not find toplevel 'tests' key")
         sys.exit(1)
     for test in jsondata['tests']:
         name = test.get("name")
         if name is None:
             sys.stderr.write("Error: Found unnamed test\n" % name)
             sys.exit(1)
         if name in names:
             sys.stderr.write("Error: Multiple tests with name '%s'\n" % name)
             sys.exit(1)
         names.add(name)
         if "metrics" not in test:
-            print "Warning: '%s' has No metrics!" % test['name']
+            print("Warning: '%s' has No metrics!" % test['name'])
             continue
         for name in test["metrics"].keys():
             if name not in columnindexes:
                 columnindexes[name] = len(columns)
                 columns.append(name)
         for name in test.keys():
             if name not in columnindexes and name in info_columns:
                 columnindexes[name] = len(columns)
                 columns.append(name)
 
     # Pass2 actual data construction
     nan = float('NaN')
     data = []
     testnames = []
     for test in jsondata['tests']:
         name = test['name']
         if 'shortname' in test:
             name = test['shortname']
         testnames.append(name)
 
         datarow = [nan] * len(columns)
         if "metrics" in test:
-            for (metricname, value) in test['metrics'].iteritems():
+            for (metricname, value) in test['metrics'].items():
                 datarow[columnindexes[metricname]] = value
-        for (name, value) in test.iteritems():
+        for (name, value) in test.items():
             index = columnindexes.get(name)
             if index is not None:
                 datarow[index] = test[name]
         data.append(datarow)
     index = pd.Index(testnames, name='Program')
     return pd.DataFrame(data=data, index=index, columns=columns)
 
 def read_report_simple_csv(filename):

 def filter_blacklist(data, blacklist):
     return data.loc[~(data.index.get_level_values(1).isin(blacklist))]
 
 def print_filter_stats(reason, before, after):
     n_before = len(before.groupby(level=1))
     n_after = len(after.groupby(level=1))
     n_filtered = n_before - n_after
     if n_filtered != 0:
-        print "%s: %s (filtered out)" % (reason, n_filtered)
+        print("%s: %s (filtered out)" % (reason, n_filtered))
 
 # Truncate a string to a maximum length by keeping a prefix, a suffix and ...
 # in the middle
 def truncate(string, prefix_len, suffix_len):
     return re.sub("^(.{%d}).*(.{%d})$" % (prefix_len, suffix_len),
                   "\g<1>...\g<2>", string)
 
 # Search for common prefixes and suffixes in a list of names and return
if shorten_names:
                 name = name[:-common_suffix]
             return "%-45s" % truncate(name, 10, 30)
 
         formatters['Program'] = lambda name: format_name(name, drop_prefix, drop_suffix)
     float_format = lambda x: "%6.2f" % (x,)
     pd.set_option("display.max_colwidth", 0)
     out = dataout.to_string(index=False, justify='left',
                             float_format=float_format, formatters=formatters)
-    print out
-    print d.describe()
+    print(out)
+    print(d.describe())
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(prog='compare.py')
     parser.add_argument('-a', '--all', action='store_true')
     parser.add_argument('-f', '--full', action='store_true')
     parser.add_argument('-m', '--metric', action='append', dest='metrics',
                         default=[])
     parser.add_argument('--nodiff', action='store_false', dest='show_diff',
for metric in metrics:
             sys.stderr.write("Unknown metric '%s'\n" % metric)
             problem = True
         if problem:
             sys.exit(1)
 
     # Filter data
     proggroup = data.groupby(level=1)
     initial_size = len(proggroup.indices)
-    print "Tests: %s" % (initial_size,)
+    print("Tests: %s" % (initial_size,))
     if config.filter_failed and hasattr(data, 'Exec'):
         newdata = filter_failed(data)
         print_filter_stats("Failed", data, newdata)
         newdata = newdata.drop('Exec', 1)
         data = newdata
     if config.filter_short:
         newdata = filter_short(data, metric)
         print_filter_stats("Short Running", data, newdata)
         data = newdata
     if config.filter_hash and 'hash' in data.columns and \
        data.index.get_level_values(0).nunique() > 1:
         newdata = filter_same_hash(data)
         print_filter_stats("Same hash", data, newdata)
         data = newdata
     if config.filter_blacklist:
         blacklist = open(config.filter_blacklist).readlines()
         blacklist = [line.strip() for line in blacklist]
         newdata = filter_blacklist(data, blacklist)
         print_filter_stats("In Blacklist", data, newdata)
         data = newdata
     final_size = len(data.groupby(level=1))
     if final_size != initial_size:
-        print "Remaining: %d" % (final_size,)
+        print("Remaining: %d" % (final_size,))
 
     # Reduce / add columns
-    print "Metric: %s" % (",".join(metrics),)
+    print("Metric: %s" % (",".join(metrics),))
     if len(metrics) > 0:
         data = data[metrics]
     data = add_diff_column(data)
 
     sortkey = 'diff'
     if len(config.files) == 1:
         sortkey = data.columns[0]
 
     # Print data
-    print ""
+    print("")
     shorten_names = not config.full
     limit_output = (not config.all) and (not config.full)
     print_result(data, limit_output, shorten_names, config.show_diff, sortkey)

test-suite/trunk/utils/tdiff.py

 
 def determine_max_commandline_len():
     """Determine maximum length of commandline possible"""
     # See also http://www.in-ulm.de/~mascheck/various/argmax/
     sc_arg_max = os.sysconf('SC_ARG_MAX')
     if sc_arg_max <= 0:
         return 10000 # wild guess
     env_len = 0
-    for key,val in os.environ.iteritems():
+    for key,val in os.environ.items():
         env_len += len(key) + len(val) + 10
     return sc_arg_max - env_len
 
 
 def get_inputs_rec(target, cwd):
     worklist = [target]
 
     result = dict()
def replace_ext(filename, newext):
     return dirname + "/" + basename.split(".", 1)[0] + newext
 
 
 def filelist(mode, target, cwd, config):
     tree = get_inputs_rec(config.target[0], cwd)
 
     if config.mode == 'sources':
         # Take leafs in the dependency tree
-        for target, depnode in tree.iteritems():
+        for target, depnode in tree.items():
             if len(depnode.inputs) == 0:
                 yield target
     else:
         # Take files ending in '.o'
-        for target, depnode in tree.iteritems():
+        for target, depnode in tree.items():
             if target.endswith(".o"):
                 # Determine .s/.stats ending used by -save-temps=obj or
                 # -save-stats=obj
                 if config.mode == 's_files':
                     target = replace_ext(target, '.s')
                 elif config.mode == 'stats':
                     target = replace_ext(target, '.stats')
                 else: