[LNT] Python 3 support: Use absolute import by default

[thopre]

Summary

Forces module import to be absolute by default on Python 2 since this is
the default for Python 3. Adapts relative imports to use the relative
import symtax 'from . import foo' or 'from .pkg import foo'.

Also remove unnecessary circular import dependency between testsuite and
testsuitedb by moving functions used by testsuite but currently in
testsuitedb into module testsuite. Finally adds type hints into
lnt.formats to avoid warnings from mypy due to change to relative import
syntax.

Changes to adapt syntax of relative import was produced by running
futurize's stage1 libfuturize.fixes.fix_absolute_import, while breaking
of circular dependency was done manually.

[thopre]

Rebase on top of Hubert Tong similar changes

[hubert.reinterpretcast]

LGTM.

lnt/trunk/lnt/formats/__init__.py

 """
 Utilities for converting to LNT's test format.
 
 LNT formats are described by dictionaries with 'name', 'read', and 'write'
 fields. Only the 'name' field is required. The 'read' field should be a
 callable taking a path_or_file object, the 'write' function should be a
 callable taking a Python object to write, and the path_or_file to write to.
 """
 
 from __future__ import absolute_import
+from typing import List, Dict
 from .PlistFormat import format as plist
 from .JSONFormat import format as json
 
-formats = [plist, json]
-formats_by_name = dict((f['name'], f) for f in formats)
-format_names = list(formats_by_name.keys())
+formats = [plist, json]  # type: List[Dict]
+formats_by_name = dict((f['name'], f) for f in formats)  # type: Dict[str, Dict]
+format_names = list(formats_by_name.keys())  # type: List[str]
 
 
 def get_format(name):
     """get_format(name) -> [format]
 
     Loookup a format object by name.
     """
 

lnt/trunk/lnt/server/db/testsuite.py

 """
 Database models for the TestSuites abstraction.
 """
 
 from __future__ import absolute_import
 import json
 import lnt
 from . import util
 
 import sqlalchemy
 import sqlalchemy.ext.declarative
 import sqlalchemy.orm
-from sqlalchemy import Column, Integer, ForeignKey, String, Binary
+from sqlalchemy import Column, Integer, ForeignKey, String, Binary, Float
 from sqlalchemy.orm import relation
 from lnt.util import logger
 
 Base = sqlalchemy.ext.declarative.declarative_base()  # type: sqlalchemy.ext.declarative.api.DeclarativeMeta
 
 
 class SampleType(Base):
     """
class StatusKind(Base):
     def __init__(self, id, name):
         self.id = id
         self.name = name
 
     def __repr__(self):
         return '%s%r' % (self.__class__.__name__, (self.name,))
 
 
+_sample_type_to_sql = {
+    'Real': Float,
+    'Hash': String,
+    'Status': Integer,
+}
+
+
+def is_known_sample_type(name):
+    return name in _sample_type_to_sql
+
+
+def make_sample_column(name, type):
+    sqltype = _sample_type_to_sql.get(type)
+    if sqltype is None:
+        raise ValueError("test suite defines unknown sample type %r" % type)
+    options = []
+    if type == 'Status':
+        options.append(ForeignKey(StatusKind.id))
+    return Column(name, sqltype, *options)
+
+
+def make_run_column(name):
+    return Column(name, String(256))
+
+
+def make_machine_column(name):
+    return Column(name, String(256))
+
+
 class _MigrationError(Exception):
     def __init__(self, message):
         full_message = \
             "Cannot automatically migrate database: %s" % message
         super(_MigrationError, self).__init__(full_message)
 
 
 class TestSuiteJSONSchema(Base):
def __init__(self, name, db_key_name):
                                   lnt.__versioninfo__[1])
 
     def __repr__(self):
         return '%s%r' % (self.__class__.__name__, (self.name, self.db_key_name,
                                                    self.version))
 
     @staticmethod
     def from_json(data):
-        from . import testsuitedb
         if data.get('format_version') != '2':
             raise ValueError("Expected \"format_version\": \"2\" in schema")
         name = data['name']
         ts = TestSuite(name=data['name'], db_key_name=data['name'])
 
         machine_fields = []
         for field_desc in data.get('machine_fields', []):
             name = field_desc['name']
def from_json(data):
         sample_fields = []
         for index, metric_desc in enumerate(data['metrics']):
             name = metric_desc['name']
             bigger_is_better = metric_desc.get('bigger_is_better', False)
             metric_type_name = metric_desc.get('type', 'Real')
             display_name = metric_desc.get('display_name')
             unit = metric_desc.get('unit')
             unit_abbrev = metric_desc.get('unit_abbrev')
-            if not testsuitedb.is_known_sample_type(metric_type_name):
+            if not is_known_sample_type(metric_type_name):
                 raise ValueError("Unknown metric type '%s'" %
                                  metric_type_name)
             metric_type = SampleType(metric_type_name)
             bigger_is_better_int = 1 if bigger_is_better else 0
             field = SampleField(name, metric_type, index, status_field=None,
                                 bigger_is_better=bigger_is_better_int,
                                 display_name=display_name, unit=unit,
                                 unit_abbrev=unit_abbrev)
class SampleField(FieldMixin, Base):
     def copy_info(self, other):
         self.display_name = other.display_name
         self.unit = other.unit
         self.unit_abbrev = other.unit_abbrev
         self.schema_index = other.schema_index
 
 
 def _upgrade_to(connectable, tsschema, new_schema, dry_run=False):
-    from . import testsuitedb
     new = json.loads(new_schema.jsonschema)
     old = json.loads(tsschema.jsonschema)
     ts_name = new['name']
     if old['name'] != ts_name:
         raise _MigrationError("Schema names differ?!?")
 
     old_metrics = {}
     for metric_desc in old.get('metrics', []):
         old_metrics[metric_desc['name']] = metric_desc
 
     for metric_desc in new.get('metrics', []):
         name = metric_desc['name']
         old_metric = old_metrics.pop(name, None)
         type = metric_desc['type']
         if old_metric is not None:
             if old_metric['type'] != type:
                 raise _MigrationError("Type mismatch in metric '%s'" %
                                       name)
         elif not dry_run:
             # Add missing columns
-            column = testsuitedb.make_sample_column(name, type)
+            column = make_sample_column(name, type)
             util.add_column(connectable, '%s_Sample' % ts_name, column)
 
     if len(old_metrics) != 0:
         raise _MigrationError("Metrics removed: %s" %
                               ", ".join(old_metrics.keys()))
 
     old_run_fields = {}
     old_order_fields = {}
for field_desc in new.get('run_fields', []):
             if old_order_field is None:
                 raise _MigrationError("Cannot add order field '%s'" %
                                       name)
             continue
 
         old_field = old_run_fields.pop(name, None)
         # Add missing columns
         if old_field is None and not dry_run:
-            column = testsuitedb.make_run_column(name)
+            column = make_run_column(name)
             util.add_column(connectable, '%s_Run' % ts_name, column)
 
     if len(old_run_fields) > 0:
         raise _MigrationError("Run fields removed: %s" %
                               ", ".join(old_run_fields.keys()))
     if len(old_order_fields) > 0:
         raise _MigrationError("Order fields removed: %s" %
                               ", ".join(old_order_fields.keys()))
 
     old_machine_fields = {}
     for field_desc in old.get('machine_fields', []):
         name = field_desc['name']
         old_machine_fields[name] = field_desc
 
     for field_desc in new.get('machine_fields', []):
         name = field_desc['name']
         old_field = old_machine_fields.pop(name, None)
         # Add missing columns
         if old_field is None and not dry_run:
-            column = testsuitedb.make_machine_column(name)
+            column = make_machine_column(name)
             util.add_column(connectable, '%s_Machine' % ts_name, column)
 
     if len(old_machine_fields) > 0:
         raise _MigrationError("Machine fields removed: %s" %
                               ", ".join(old_machine_fields.keys()))
     # The rest should just be metadata that we can upgrade
     return True
 

lnt/trunk/lnt/server/db/testsuitedb.py

 def _dict_update_abort_on_duplicates(base_dict, to_merge):
     '''This behaves like base_dict.update(to_merge) but asserts that none
     of the keys in to_merge is present in base_dict yet.'''
     for key, value in to_merge.items():
         assert base_dict.get(key, None) is None
         base_dict[key] = value
 
 
-_sample_type_to_sql = {
-    'Real': Float,
-    'Hash': String,
-    'Status': Integer,
-}
-
-
-def is_known_sample_type(name):
-    return name in _sample_type_to_sql
-
-
-def make_sample_column(name, type):
-    sqltype = _sample_type_to_sql.get(type)
-    if sqltype is None:
-        raise ValueError("test suite defines unknown sample type %r" % type)
-    options = []
-    if type == 'Status':
-        options.append(ForeignKey(testsuite.StatusKind.id))
-    return Column(name, sqltype, *options)
-
-
-def make_run_column(name):
-    return Column(name, String(256))
-
-
-def make_machine_column(name):
-    return Column(name, String(256))
-
-
 class MachineInfoChanged(ValueError):
     pass
 
 
 class TestSuiteDB(object):
     """
     Wrapper object for an individual test suites database tables.
 
def __init__(self, v4db, name, test_suite):
             # machine fields.
             class_dict = locals()
             for item in fields:
                 iname = item.name
                 if iname in class_dict:
                     raise ValueError("test suite defines reserved key %r" % (
                         iname))
 
-                class_dict[iname] = item.column = make_machine_column(iname)
+                item.column = testsuite.make_machine_column(iname)
+                class_dict[iname] = item.column
 
             def __init__(self, name_value):
                 self.id = None
                 self.name = name_value
 
             def __repr__(self):
                 return '%s_%s%r' % (db_key_name, self.__class__.__name__,
                                     (self.id, self.name))
def __init__(self, v4db, name, test_suite):
             # but need a bit for that in the test suite definition.
             class_dict = locals()
             for item in fields:
                 iname = item.name
                 if iname in class_dict:
                     raise ValueError("test suite defines reserved key %r" %
                                      (iname,))
 
-                class_dict[iname] = item.column = make_run_column(iname)
+                item.column = testsuite.make_run_column(iname)
+                class_dict[iname] = item.column
 
             def __init__(self, new_id, machine, order, start_time, end_time):
                 self.id = new_id
                 self.machine = machine
                 self.order = order
                 self.start_time = start_time
                 self.end_time = end_time
                 self.imported_from = None
def __init__(self, v4db, name, test_suite):
             # the new UI is up.
             class_dict = locals()
             for item in self.sample_fields:
                 iname = item.name
                 if iname in class_dict:
                     raise ValueError("test suite defines reserved key %r" %
                                      (iname,))
 
-                item.column = make_sample_column(iname, item.type.name)
+                item.column = testsuite.make_sample_column(iname,
+                                                           item.type.name)
                 class_dict[iname] = item.column
 
             def __init__(self, run, test, **kwargs):
                 self.id = None
                 self.run = run
                 self.test = test
 
                 # Initialize sample fields (defaulting to 0, for now).

lnt/trunk/lnt/tests/nt.py

 """LLVM test-suite compile and execution tests"""
+from __future__ import absolute_import
 from __future__ import print_function
 import csv
 import os
 import platform
 import re
 import shutil
 import subprocess
 import sys

 from lnt.testing.util.rcs import get_source_version
 
 from lnt.testing.util.misc import timestamp
 
 from lnt.server.reporting.analysis import UNCHANGED_PASS, UNCHANGED_FAIL
 from lnt.server.reporting.analysis import REGRESSED, IMPROVED
 from lnt.util import logger
 from lnt.lnttool.common import submit_options
-import builtintest
+from . import builtintest
 
 
 class TestModule(object):
     """
     Base class for extension test modules.
     """
 
     def __init__(self):