[LNT] Python 3 support: use Python 3 module names

[thopre]

Summary

Adapt imports to use Python 3 module names by installing them as
aliases. Due to the clear separation between text and binary data in
Python 3, this also requires changing all I/O in profilev2impl to be
binary I/O with encoding/decoding for text and some adaptation to the
integer to ULEB conversion code. Installing aliases was produced by
running futurize's stage2 libfuturize.fixes.fix_future_standard_library
and further edited by hand to adapt that I/O code.

[thopre]

Ping?

[thopre]

• Fix module name in global statement
• Rebase on top of global import fix

[thopre]

Ping?

[thopre]

Ping?

[thopre]

Ping?

[thopre]

Ping?

[thopre]

Rebase later in the patch series

[PrzemekWirkus]

lnt/lnttool/admin.py

10–16

This idiom was discussed in other patch, let us know how you will sort this thing out please Thomas.

[thopre]

Rebase on top of latest changes in earlier patches

[PrzemekWirkus]

LGTM

[thopre]

Rebase earlier in patch series

[thopre]

Committed as d935e954283fac46ac47e9b0e87fd6ecab6ce64d

lnt/lnttool/admin.py

 #!/usr/bin/env python
+from future import standard_library
+standard_library.install_aliases()
 import click
 from importlib import import_module
 from .common import submit_options
 
 
 def _load_dependencies():
-    global yaml, sys, requests, json, os, httplib
+    global yaml, sys, requests, json, os, http
     yaml = import_module('yaml')
     sys = import_module('sys')
     requests = import_module('requests')
     json = import_module('json')
     os = import_module('os')
-    httplib = import_module('httplib')
+    http = import_module('http.client')
 
 
 def _error(msg):
     sys.stderr.write('%s\n' % msg)
 
 
 def _fatal(msg):
     _error(msg)

 def _check_response(response):
     '''Check given response. If it is not a 200 response print an error message
     and quit.'''
     status_code = response.status_code
     if 200 <= status_code and status_code < 400:
         return
 
     sys.stderr.write("%d: %s\n" %
-                     (status_code, httplib.responses.get(status_code, '')))
+                     (status_code, http.client.responses.get(status_code, '')))
     sys.stderr.write("\n%s\n" % response.text)
     sys.exit(1)
 
 
 def _sorted_dict_items(dict, *args):
     result = list()
     for key in args:
         if key in dict:

lnt/lnttool/viewcomparison.py

def action_view_comparison(report_a, report_b, hostname, port, dry_run,
     import contextlib
     import lnt.server.db.migrate
     import lnt.server.instance
     import lnt.server.ui.app
     import logging
     import os
     import shutil
     import tempfile
-    import thread
+    import _thread
 
     init_logger(logging.ERROR)
 
     # Create a temporary directory to hold the instance.
     tmpdir = tempfile.mkdtemp(suffix='lnt')
 
     try:
         # Create a temporary instance.
try:
             import_and_report(config, 'default', db, session, report_b,
                               '<auto>', testsuite, select_machine='match')
 
             # Dispatch another thread to start the webbrowser.
             comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
             logger.info("opening comparison view: %s" % (comparison_url,))
 
             if not dry_run:
-                thread.start_new_thread(_start_browser, (comparison_url, True))
+                _thread.start_new_thread(_start_browser, (comparison_url, True))
 
             # Run the webserver.
             app = lnt.server.ui.app.App.create_with_instance(instance)
             app.debug = True
 
             if dry_run:
                 # Don't catch out exceptions.
                 app.testing = True
                 # Create a test client.
                 client = app.test_client()
                 response = client.get(comparison_url)
                 assert response.status_code == 200, "Page did not return 200."
             else:
                 app.run(hostname, port, use_reloader=False)
     finally:
         shutil.rmtree(tmpdir)

lnt/server/ui/app.py

 from __future__ import print_function
-import StringIO
+from future import standard_library
+standard_library.install_aliases()
+import io
 import logging
 import logging.handlers
 import sys
 import time
 import traceback
 from logging import Formatter
 
 import datetime
class Request(flask.Request):
     # Utility Methods
     def get_db(self):
         assert self.db is not None
         # Enable SQL logging with db_log.
         #
         # FIXME: Conditionalize on an is_production variable.
         show_sql = bool(self.args.get('db_log') or self.form.get('db_log'))
         if show_sql:
-            g.db_log = StringIO.StringIO()
+            g.db_log = io.StringIO()
             logger = logging.getLogger("sqlalchemy")
             logger.addHandler(logging.StreamHandler(g.db_log))
         return self.db
 
     def get_testsuite(self):
         """
         get_testsuite() -> server.db.testsuite.TestSuite
 

lnt/server/ui/filters.py

 from future import standard_library
 standard_library.install_aliases()
-import StringIO
+import io
 import datetime
 import pprint
 import urllib.parse
 import time
 
 from lnt.server.ui import util
 
 

 
 def filter_aspctcell(value, class_=None, style=None, attributes=None, *args,
                      **kwargs):
     cell = util.PctCell(value, *args, **kwargs)
     return cell.render(class_, style, attributes)
 
 
 def filter_pprint(value):
-    stream = StringIO.StringIO()
+    stream = io.StringIO()
     pprint.pprint(value, stream)
     return stream.getvalue()
 
 
 def filter_format_or_default(fmt, input, default):
     if input:
         return fmt % input
     else:

lnt/server/ui/views.py

+from future import standard_library
+standard_library.install_aliases()
 import datetime
 import json
 import os
 import re
 import time
 from collections import namedtuple, defaultdict
-from urlparse import urlparse, urljoin
+from urllib.parse import urlparse, urljoin
 
 import flask
 import sqlalchemy.sql
 from flask import abort
 from flask import current_app
 from flask import flash
 from flask import g
 from flask import make_response

lnt/testing/profile/profilev1impl.py

+from future import standard_library
+standard_library.install_aliases()
 from lnt.testing.profile.profile import ProfileImpl
-import cPickle
+import pickle
 import zlib
 
 
 class ProfileV1(ProfileImpl):
     """
 ProfileV1 files not clever in any way. They are simple Python objects with
 the profile data layed out in the most obvious way for production/consumption
 that are then pickled and compressed.
functions: {
     @staticmethod
     def checkFile(fn):
         # "zlib compressed data" - 78 9C
         return open(fn, 'rb').read(2) == b'\x78\x9c'
 
     @staticmethod
     def deserialize(fobj):
         o = zlib.decompress(fobj.read())
-        data = cPickle.loads(o)
+        data = pickle.loads(o)
         return ProfileV1(data)
 
     def serialize(self, fname=None):
-        obj = cPickle.dumps(self.data)
+        obj = pickle.dumps(self.data)
         compressed_obj = zlib.compress(obj)
 
         if fname is None:
             return bytes(compressed_obj)
         else:
             with open(fname, 'w') as fd:
                 fd.write(compressed_obj)
 

lnt/testing/profile/profilev2impl.py

 from __future__ import absolute_import
 from builtins import range
+from future import standard_library
+standard_library.install_aliases()
 from .profile import ProfileImpl
-import StringIO
 import bz2
 import copy
 import io
 import os
 import struct
 
 """
 ProfileV2 is a profile data representation designed to keep the

 
 def readNum(fobj):
     """
     Reads a ULEB encoded number from a stream.
     """
     n = 0
     shift = 0
     while True:
-        b = ord(fobj.read(1))
+        b = bytearray(fobj.read(1))[0]
         n |= (b & 0x7F) << shift
         shift += 7
         if (b & 0x80) == 0:
             return n
 
 
 def writeNum(fobj, n):
     """
     Write 'n' as a ULEB encoded number to a stream.
     """
     while True:
         b = n & 0x7F
         n >>= 7
         if n != 0:
             b |= 0x80
-        fobj.write(chr(b))
+        fobj.write(bytearray([b]))
 
         if n == 0:
             break
 
 
 def readString(fobj):
     """
     Read a string from a stream.
     """
-    return fobj.readline()[:-1]
+    return fobj.readline()[:-1].decode()
 
 
 def writeString(fobj, s):
     """
     Write a string to a stream.
     """
-    fobj.write(str(s))
-    fobj.write('\n')
+    fobj.write(s.encode())
+    fobj.write(u'\n'.encode())
 
 
 def readFloat(fobj):
     """
     Read a floating point number from a stream.
     """
     num = readNum(fobj)
     packed = struct.pack('>l', num)
class Section(object):
 
     def copy(self):
         return copy.copy(self)
 
 
 class CompressedSection(Section):
     def read(self, fobj):
         fobj.seek(self.offset + self.start)
-        _io = StringIO.StringIO(bz2.decompress(fobj.read(self.size)))
+        _io = io.BytesIO(bz2.decompress(fobj.read(self.size)))
         return self.deserialize(_io)
 
     def write(self, fobj):
         _io = io.BytesIO()
         self.serialize(_io)
         fobj.write(bz2.compress(_io.getvalue()))
 
 
def writeHeader(self, fobj, offset, size):
         Section.writeHeader(self, fobj, offset, size)
         writeString(fobj, self.pool_fname)
 
     def read(self, fobj):
         if self.pool_fname:
             raise NotImplementedError()
 
         else:
-            _io = StringIO.StringIO(bz2.decompress(fobj.read(self.size)))
+            _io = io.BytesIO(bz2.decompress(fobj.read(self.size)))
             self.size = len(_io.getvalue())
             return self.deserialize(_io)
 
     def write(self, fobj):
-        _io = StringIO.StringIO()
+        _io = io.BytesIO()
         if self.pool_fname:
             raise NotImplementedError()
 
         else:
             Section.write(self, _io)
             fobj.write(bz2.compress(_io.getvalue()))
 
 ##############################################################################
class LineCounters(CompressedSection):
     def getOffsetFor(self, fname):
         return self.function_offsets[fname]
 
     def setOffsetFor(self, fname, value):
         self.function_offsets[fname] = value
 
     def extractForFunction(self, fname, counters):
         offset = self.function_offsets[fname]
-        _io = StringIO.StringIO(self.data)
+        _io = io.BytesIO(self.data)
         _io.seek(offset)
         counters.sort()
         while True:
             c = {}
             for k in counters:
                 c[k] = readFloat(_io)
             yield c
 
class LineAddresses(CompressedSection):
     def getOffsetFor(self, fname):
         return self.function_offsets[fname]
 
     def setOffsetFor(self, fname, value):
         self.function_offsets[fname] = value
 
     def extractForFunction(self, fname):
         offset = self.function_offsets[fname]
-        _io = StringIO.StringIO(self.data)
+        _io = io.BytesIO(self.data)
         _io.seek(offset)
         last_address = 0
         while True:
             address = readNum(_io) + last_address
             last_address = address
             yield address
 
 
def getOffsetFor(self, fname):
         return self.function_offsets[fname]
 
     def setOffsetFor(self, fname, value):
         self.function_offsets[fname] = value
 
     def extractForFunction(self, fname):
         offset = self.function_offsets[fname]
 
-        _io = StringIO.StringIO(self.data)
+        _io = io.BytesIO(self.data)
         _io.seek(offset)
         while True:
             n = readNum(_io)
             yield self.text_pool.getAt(n)
 
     def copy(self, tp):
         new = copy.copy(self)
         new.text_pool = tp
         return new
 
 
 class TextPool(MaybePooledSection):
     def __init__(self):
         MaybePooledSection.__init__(self)
         self.offsets = {}
         # Populate data with a single character initially so that zero is
         # never a valid string pool index. LineText relies upon this to use
         # zero as a sentinel.
-        self.data = StringIO.StringIO('\n')
+        self.data = io.BytesIO(u'\n'.encode())
         self.pool_read = False
 
     def serialize(self, fobj):
         self.data.seek(0)
         fobj.write(self.data.read())
 
     def deserialize(self, fobj):
         # FIXME: Make this lazy!
-        self.data = StringIO.StringIO(fobj.read(self.size))
+        self.data = io.BytesIO(fobj.read(self.size))
 
     def upgrade(self, impl):
         pass
 
     def getOrCreate(self, text):
         if self.pool_fname and not self.pool_read:
             assert False
             self.readFromPool()
def deserialize(fobj):
         for section in p.sections:
             section.read(fobj)
 
         return p
 
     def serialize(self, fname=None):
         # If we're not writing to a file, emulate a file object instead.
         if fname is None:
-            fobj = StringIO.StringIO()
+            fobj = io.BytesIO()
         else:
             fobj = open(fname, 'wb')
 
         # Take a copy of all sections. While writing we may change
         # offsets / indices, and we need to ensure we can modify our
         # sections' states without affecting the original object (we may
         # need to read from it while writing! (getCodeForFunction))
         h = self.h.copy()
         cnp = self.cnp.copy()
         tlc = self.tlc.copy(cnp)
         lc = self.lc.copy()
         la = self.la.copy()
         tp = self.tp.copy()
         lt = self.lt.copy(tp)
         f = self.f.copy(cnp, lc, la, lt)
         sections = [h, cnp, tlc, lc, la, lt, tp, f]
 
         writeNum(fobj, 2)  # Version
 
         # We need to write all sections first, so we know their offset
         # before we write the header.
-        tmpio = StringIO.StringIO()
+        tmpio = io.BytesIO()
         offsets = {}
         sizes = {}
         for section in sections:
             offsets[section] = tmpio.tell()
             section.write(tmpio)
             sizes[section] = tmpio.tell() - offsets[section]
 
         for section in sections:

lnt/testing/util/rcs.py

+from future import standard_library
+standard_library.install_aliases()
 import re
 import os
 from lnt.util import logger
 from lnt.testing.util import commands
 
 _git_svn_id_re = re.compile("^    git-svn-id: [^@]*@([0-9]+) .*$")
 
 

lnt/tests/compile.py

 """Single file compile-time performance testing"""
 from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 import errno
 import hashlib
 import json
 import os
 import platform
 import pprint
 import re
 import shlex

lnt/util/wsgi_restart.py

 # This code lifted from the mod_wsgi docs.
 from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 import os
 import sys
 import signal
 import threading
 import atexit
-import Queue
+import queue
 
 _interval = 1.0
 _times = {}
 _files = []
 
 _running = False
-_queue = Queue.Queue()
+_queue = queue.Queue()
 _lock = threading.Lock()
 
 
 def _restart(path):
     _queue.put(True)
     prefix = 'monitor (pid=%d):' % os.getpid()
     print('%s Change detected to \'%s\'.' % (prefix, path), file=sys.stderr)
     print('%s Triggering process restart.' % prefix, file=sys.stderr)

tests/server/ui/V4Pages.py

 # Import a profile
 # RUN: lnt import %t.instance %{shared_inputs}/profile-report.json
 #
 # RUN: lnt import %t.instance %S/Inputs/sample-failed-report1.json
 # RUN: lnt import %t.instance %S/Inputs/sample-failed-report2.json
 #
 # RUN: python %s %t.instance %{tidylib}
 
+from future import standard_library
+standard_library.install_aliases()
 import logging
 import re
 import sys
 import xml.etree.ElementTree as ET
-from htmlentitydefs import name2codepoint
+from html.entities import name2codepoint
 from flask import session
 import lnt.server.db.migrate
 import lnt.server.ui.app
 import json
 
 # We can validate html if pytidylib is available and tidy-html5 is installed.
 # The user can indicate this by passing --use-tidylib to the script (triggered
 # by `lit -Dtidylib=1`)