[LNT] Fix postgres unit tests by closing db connections.

[kristof.beyls]

Summary

When running the unit tests on a postgres database, the following error shows:

sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30

We've also seen this error from time to time on the instance running at
llvm.org/perf, and it was unclear what the exact reason was for this. Now that
we can run the unit tests on postgres, this shows that not closing the db
connection at the end of a request is at least one of the reasons this error
can be produced.

This builds on http://reviews.llvm.org/D13163, which makes it possible to run the unit tests on postgres instead of sqlite.

[cmatthews]

This looks good to me.

lnt/trunk/lnt/lnttool/import_data.py

 import os, pprint, sys, time
 
 import lnt.formats
 import lnt.util.ImportData
 import lnt.server.instance
+import contextlib
+
 
 def action_import(name, args):
     """import test data into a database"""
 
     from optparse import OptionParser, OptionGroup
 
     parser = OptionParser("%s [options] <instance path> <file>+"%name)
     parser.add_option("", "--database", dest="database", default="default",
def action_import(name, args):
 
     path = args.pop(0)
 
     # Load the LNT instance.
     instance = lnt.server.instance.Instance.frompath(path)
     config = instance.config
 
     # Get the database.
-    db = config.get_database(opts.database, echo=opts.show_sql)
-
+    with contextlib.closing(config.get_database(opts.database,
+                                                echo=opts.show_sql)) as db:
         # Load the database.
         success = True
         for file in args:
             result = lnt.util.ImportData.import_and_report(
                 config, opts.database, db, file,
                 opts.format, opts.commit, opts.show_sample_count,
                 opts.no_email, opts.no_report)
 
             success &= result.get('success', False)
             if opts.quiet:
                 continue
 
             if opts.show_raw_result:
                 pprint.pprint(result)
             else:
                 lnt.util.ImportData.print_report_result(result, sys.stdout,
                                                         sys.stderr,
                                                         opts.verbose)
 
         if not success:
             raise SystemExit, 1
-

lnt/trunk/lnt/lnttool/main.py

 """Implement the command line 'lnt' tool."""
 
 import logging
 import os
 import sys
 import tempfile
 from optparse import OptionParser, OptionGroup
+import contextlib
 
 import werkzeug.contrib.profiler
 
 import StringIO
 import lnt
 import lnt.util.multitool
 import lnt.util.ImportData
 from lnt import testing
def action_send_daily_report(name, args):
 
     path, to_address = args
 
     # Load the LNT instance.
     instance = lnt.server.instance.Instance.frompath(path)
     config = instance.config
 
     # Get the database.
-    db = config.get_database(opts.database)
+    with contextlib.closing(config.get_database(opts.database)) as db:
 
         # Get the testsuite.
         ts = db.testsuite[opts.testsuite]
 
         if opts.today:
             date = datetime.datetime.utcnow()
         else:
             # Get a timestamp to use to derive the daily report to generate.
             latest = ts.query(ts.Run).\
                 order_by(ts.Run.start_time.desc()).limit(1).first()
 
-        # If we found a run, use it's start time (rounded up to the next hour,
-        # so we make sure it gets included).
+            # If we found a run, use it's start time (rounded up to the next
+            # hour, so we make sure it gets included).
             if latest:
                 date = latest.start_time + datetime.timedelta(hours=1)
             else:
                 # Otherwise, just use now.
                 date = datetime.datetime.utcnow()
 
         # Generate the daily report.
         note("building report data...")
         report = lnt.server.reporting.dailyreport.DailyReport(
             ts, year=date.year, month=date.month, day=date.day,
             day_start_offset_hours=date.hour, for_mail=True,
             num_prior_days_to_include=opts.days,
             filter_machine_regex=opts.filter_machine_regex)
         report.build()
 
         note("generating HTML report...")
-    ts_url = "%s/db_%s/v4/%s" % (config.zorgURL, opts.database, opts.testsuite)
+        ts_url = "%s/db_%s/v4/%s" \
+            % (config.zorgURL, opts.database, opts.testsuite)
         subject = "Daily Report: %04d-%02d-%02d" % (
             report.year, report.month, report.day)
         html_report = report.render(ts_url, only_html_body=False)
 
         if opts.subject_prefix is not None:
             subject = "%s %s" % (opts.subject_prefix, subject)
 
         # Form the multipart email message.
         msg = email.mime.multipart.MIMEMultipart('alternative')
         msg['Subject'] = subject
         msg['From'] = opts.from_address
         msg['To'] = to_address
         msg.attach(email.mime.text.MIMEText(html_report, "html"))
 
         # Send the report.
         if not opts.dry_run:
             s = smtplib.SMTP(opts.host)
             s.sendmail(opts.from_address, [to_address],
                        msg.as_string())
             s.quit()
 
+
 def action_send_run_comparison(name, args):
     """send a run-vs-run comparison email"""
     import email.mime.multipart
     import email.mime.text
     import smtplib
 
     import lnt.server.reporting.dailyreport
     
handler.setFormatter(logging.Formatter(
             datefmt='%Y-%m-%d %H:%M:%S'))
     logger.addHandler(handler)
 
     # Load the LNT instance.
     instance = lnt.server.instance.Instance.frompath(path)
     config = instance.config
 
     # Get the database.
-    db = config.get_database(opts.database)
+    with contextlib.closing(config.get_database(opts.database)) as db:
 
         # Get the testsuite.
         ts = db.testsuite[opts.testsuite]
 
         # Lookup the two runs.
         run_a_id = int(run_a_id)
         run_b_id = int(run_b_id)
         run_a = ts.query(ts.Run).\
             filter_by(id=run_a_id).first()
         run_b = ts.query(ts.Run).\
             filter_by(id=run_b_id).first()
         if run_a is None:
             parser.error("invalid run ID %r (not in database)" % (run_a_id,))
         if run_b is None:
             parser.error("invalid run ID %r (not in database)" % (run_b_id,))
 
         # Generate the report.
         reports = lnt.server.reporting.runs.generate_run_report(
             run_b, baseurl=config.zorgURL, only_html_body=False, result=None,
             compare_to=run_a, baseline=None,
             aggregation_fn=min)
         subject, text_report, html_report, _ = reports
 
         if opts.subject_prefix is not None:
             subject = "%s %s" % (opts.subject_prefix, subject)
 
         # Form the multipart email message.
         msg = email.mime.multipart.MIMEMultipart('alternative')
         msg['Subject'] = subject
         msg['From'] = opts.from_address
         msg['To'] = opts.to_address
         msg.attach(email.mime.text.MIMEText(text_report, 'plain'))
         msg.attach(email.mime.text.MIMEText(html_report, 'html'))
 
         # Send the report.
         if not opts.dry_run:
             s = smtplib.SMTP(opts.host)
             s.sendmail(opts.from_address, [opts.to_address],
                        msg.as_string())
             s.quit()
 
 ###
 
 def _version_check():
     """
     Check that the installed version of the LNT is up-to-date with the running
     package.
 

lnt/trunk/lnt/lnttool/updatedb.py

 import os
 from optparse import OptionParser, OptionGroup
+import contextlib
 
 import lnt.server.instance
 from lnt.testing.util.commands import note, warning, error, fatal
 
 def action_updatedb(name, args):
     """modify a database"""
 
     from optparse import OptionParser, OptionGroup
parser.add_option("", "--delete-run", dest="delete_runs",
                       action="append", default=[], type=int)
     (opts, args) = parser.parse_args(args)
 
     if len(args) != 1:
         parser.error("invalid number of arguments")
 
     if opts.testsuite is None:
         parser.error("--testsuite is required")
-        
     path, = args
 
     # Load the instance.
     instance = lnt.server.instance.Instance.frompath(path)
 
     # Get the database and test suite.
-    db = instance.get_database(opts.database, echo=opts.show_sql)
+    with contextlib.closing(instance.get_database(opts.database,
+                                                  echo=opts.show_sql)) as db:
         ts = db.testsuite[opts.testsuite]
 
         # Compute a list of all the runs to delete.
         runs_to_delete = list(opts.delete_runs)
         if opts.delete_machines:
             runs_to_delete.extend(
                 id
                 for id, in ts.query(ts.Run.id).\
                     join(ts.Machine).\
                     filter(ts.Machine.name.in_(opts.delete_machines)))
-        
         # Delete all samples associated with those runs.
         ts.query(ts.Sample).\
             filter(ts.Sample.run_id.in_(runs_to_delete)).\
             delete(synchronize_session=False)
 
         # Delete all those runs.
         ts.query(ts.Run).\
             filter(ts.Run.id.in_(runs_to_delete)).\
             delete(synchronize_session=False)
 
         # Delete the machines.
         for name in opts.delete_machines:
             # Delete all FieldChanges associated with this machine.
             ids = ts.query(ts.FieldChange.id).\
                 join(ts.Machine).filter(ts.Machine.name == name).all()
             for i in ids:
-            ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).delete()
+                ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).\
+                    delete()
 
             num_deletes = ts.query(ts.Machine).filter_by(name=name).delete()
             if num_deletes == 0:
                 warning("unable to find machine named: %r" % name)
 
         if opts.commit:
             db.commit()
         else:
             db.rollback()

lnt/trunk/lnt/lnttool/viewcomparison.py

 import logging
 import os
 import shutil
 import sys
 import tempfile
 import thread
 import time
 import urllib
 import webbrowser
 from optparse import OptionParser, OptionGroup
+import contextlib
 
 import lnt.util.ImportData
 from lnt.testing.util.commands import note, warning, error, fatal
 
 def start_browser(url, debug=False):
     def url_is_up(url):
         try:
             o = urllib.urlopen(url)
try:
             'LNT', url, db_path, tmpdir,
             None, { 'default' : db_info })
         instance = lnt.server.instance.Instance(None, config)
 
         # Create the database.
         lnt.server.db.migrate.update_path(db_path)
 
         # Import the two reports.
-        db = config.get_database('default')
+        with contextlib.closing(config.get_database('default')) as db:
             result = lnt.util.ImportData.import_and_report(
                 config, 'default', db, report_a_path,
                 '<auto>', commit=True)
             result = lnt.util.ImportData.import_and_report(
                 config, 'default', db, report_b_path,
                 '<auto>', commit=True)
 
             # Dispatch another thread to start the webbrowser.
             comparison_url = '%s/v4/nts/2?compare_to=1' % (url,)
             note("opening comparison view: %s" % (comparison_url,))
-        thread.start_new_thread(start_browser, (comparison_url,True))
+            thread.start_new_thread(start_browser, (comparison_url, True))
 
             # Run the webserver.
             app = lnt.server.ui.app.App.create_with_instance(instance)
             app.debug = True
             app.run(opts.hostname, opts.port, use_reloader=False)
     finally:
         shutil.rmtree(tmpdir)

lnt/trunk/lnt/server/db/v4db.py

def __init__(self, path, config, baseline_revision=0, echo=False):
         # Resolve or create the known sample types.
         self.real_sample_type = self.query(testsuite.SampleType)\
             .filter_by(name = "Real").first()
         self.status_sample_type = self.query(testsuite.SampleType)\
             .filter_by(name = "Status").first()
         assert (self.real_sample_type and self.status_sample_type), \
             "sample types not initialized!"
 
+    def close(self):
+        if self.session is not None:
+            self.session.close()
+
     @property
     def testsuite(self):
         # This is the start of "magic" part of V4DB, which allows us to get
         # fully bound SA instances for databases which are effectively described
         # by the TestSuites table.
 
         # The magic starts by returning a object which will allow us to use
         # dictionary like access to get the per-test suite database wrapper.

lnt/trunk/lnt/server/ui/app.py

def get_testsuite(self):
             testsuites = self.get_db().testsuite
             if g.testsuite_name not in testsuites:
                 flask.abort(404)
 
             self.testsuite = testsuites[g.testsuite_name]
 
         return self.testsuite
 
+    def close(self):
+        db = getattr(self, 'db', None)
+        if db is not None:
+            db.close()
+        return super(Request, self).close()
+
+
 class App(flask.Flask):
     @staticmethod
     def create_with_instance(instance):
         # Construct the application.
         app = App(__name__)
 
         # Register additional filters.
         create_jinja_environment(app.jinja_env)

lnt/trunk/lnt/util/ImportData.py

def import_and_report(config, db_name, db, file, format, commit=False,
     result['total_time'] = time.time() - startTime
 
     # If this database has a shadow import configured, import the run into that
     # database as well.
     if config and config.databases[db_name].shadow_import:
         # Load the shadow database to import into.
         db_config = config.databases[db_name]
         shadow_name = db_config.shadow_import
-        shadow_db = config.get_database(shadow_name)
+        with closing(config.get_database(shadow_name)) as shadow_db:
             if shadow_db is None:
-            raise ValueError,("invalid configuration, shadow import "
+                raise ValueError, ("invalid configuration, shadow import "
                                    "database %r does not exist") % shadow_name
 
             # Perform the shadow import.
             shadow_result = import_and_report(config, shadow_name,
                                               shadow_db, file, format, commit,
                                               show_sample_count, disable_email,
                                               disable_report)
 
             # Append the shadow result to the result.
             result['shadow_result'] = shadow_result
 
     result['success'] = True
     return result
 
 def print_report_result(result, out, err, verbose = True):
     """
     print_report_result(result, out, [err], [verbose]) -> None
 

lnt/trunk/lnt/util/ServerUtil.py

except:
         return
 
 def submitFileToInstance(path, file, commit):
     # Otherwise, assume it is a local url and submit to the default database
     # in the instance.
     instance = lnt.server.instance.Instance.frompath(path)
     config = instance.config
     db_name = 'default'
-    db = config.get_database(db_name)
+    with closing(config.get_database(db_name)) as db:
         if db is None:
             raise ValueError("no default database in instance: %r" % (path,))
         return lnt.util.ImportData.import_and_report(
             config, db_name, db, file, format='<auto>', commit=commit)
 
+
 def submitFile(url, file, commit, verbose):
     # If this is a real url, submit it using urllib.
     if '://' in url:
         result = submitFileToServer(url, file, commit)
         if result is None:
             return
     else:
         result = submitFileToInstance(url, file, commit)
     return result
 
 def submitFiles(url, files, commit, verbose):
     results = []
     for file in files:
         result = submitFile(url, file, commit, verbose)
         results.append(result)
     return results