Index: lnt/lnttool/import_data.py =================================================================== --- lnt/lnttool/import_data.py +++ lnt/lnttool/import_data.py @@ -3,6 +3,8 @@ import lnt.formats import lnt.util.ImportData import lnt.server.instance +import contextlib + def action_import(name, args): """import test data into a database""" @@ -45,27 +47,26 @@ config = instance.config # Get the database. - db = config.get_database(opts.database, echo=opts.show_sql) - - # Load the database. - success = True - for file in args: - result = lnt.util.ImportData.import_and_report( - config, opts.database, db, file, - opts.format, opts.commit, opts.show_sample_count, - opts.no_email, opts.no_report) - - success &= result.get('success', False) - if opts.quiet: - continue + with contextlib.closing(config.get_database(opts.database, + echo=opts.show_sql)) as db: + # Load the database. + success = True + for file in args: + result = lnt.util.ImportData.import_and_report( + config, opts.database, db, file, + opts.format, opts.commit, opts.show_sample_count, + opts.no_email, opts.no_report) - if opts.show_raw_result: - pprint.pprint(result) - else: - lnt.util.ImportData.print_report_result(result, sys.stdout, - sys.stderr, - opts.verbose) + success &= result.get('success', False) + if opts.quiet: + continue - if not success: - raise SystemExit, 1 + if opts.show_raw_result: + pprint.pprint(result) + else: + lnt.util.ImportData.print_report_result(result, sys.stdout, + sys.stderr, + opts.verbose) + if not success: + raise SystemExit, 1 Index: lnt/lnttool/main.py =================================================================== --- lnt/lnttool/main.py +++ lnt/lnttool/main.py @@ -5,6 +5,7 @@ import sys import tempfile from optparse import OptionParser, OptionGroup +import contextlib import werkzeug.contrib.profiler @@ -293,57 +294,59 @@ config = instance.config # Get the database. - db = config.get_database(opts.database) + with contextlib.closing(config.get_database(opts.database)) as db: - # Get the testsuite. - ts = db.testsuite[opts.testsuite] + # Get the testsuite. + ts = db.testsuite[opts.testsuite] - if opts.today: - date = datetime.datetime.utcnow() - else: - # Get a timestamp to use to derive the daily report to generate. - latest = ts.query(ts.Run).\ - order_by(ts.Run.start_time.desc()).limit(1).first() - - # If we found a run, use it's start time (rounded up to the next hour, - # so we make sure it gets included). - if latest: - date = latest.start_time + datetime.timedelta(hours=1) - else: - # Otherwise, just use now. + if opts.today: date = datetime.datetime.utcnow() + else: + # Get a timestamp to use to derive the daily report to generate. + latest = ts.query(ts.Run).\ + order_by(ts.Run.start_time.desc()).limit(1).first() + + # If we found a run, use it's start time (rounded up to the next + # hour, so we make sure it gets included). + if latest: + date = latest.start_time + datetime.timedelta(hours=1) + else: + # Otherwise, just use now. + date = datetime.datetime.utcnow() + + # Generate the daily report. + note("building report data...") + report = lnt.server.reporting.dailyreport.DailyReport( + ts, year=date.year, month=date.month, day=date.day, + day_start_offset_hours=date.hour, for_mail=True, + num_prior_days_to_include=opts.days, + filter_machine_regex=opts.filter_machine_regex) + report.build() + + note("generating HTML report...") + ts_url = "%s/db_%s/v4/%s" \ + % (config.zorgURL, opts.database, opts.testsuite) + subject = "Daily Report: %04d-%02d-%02d" % ( + report.year, report.month, report.day) + html_report = report.render(ts_url, only_html_body=False) + + if opts.subject_prefix is not None: + subject = "%s %s" % (opts.subject_prefix, subject) + + # Form the multipart email message. + msg = email.mime.multipart.MIMEMultipart('alternative') + msg['Subject'] = subject + msg['From'] = opts.from_address + msg['To'] = to_address + msg.attach(email.mime.text.MIMEText(html_report, "html")) + + # Send the report. + if not opts.dry_run: + s = smtplib.SMTP(opts.host) + s.sendmail(opts.from_address, [to_address], + msg.as_string()) + s.quit() - # Generate the daily report. - note("building report data...") - report = lnt.server.reporting.dailyreport.DailyReport( - ts, year=date.year, month=date.month, day=date.day, - day_start_offset_hours=date.hour, for_mail=True, - num_prior_days_to_include=opts.days, - filter_machine_regex=opts.filter_machine_regex) - report.build() - - note("generating HTML report...") - ts_url = "%s/db_%s/v4/%s" % (config.zorgURL, opts.database, opts.testsuite) - subject = "Daily Report: %04d-%02d-%02d" % ( - report.year, report.month, report.day) - html_report = report.render(ts_url, only_html_body=False) - - if opts.subject_prefix is not None: - subject = "%s %s" % (opts.subject_prefix, subject) - - # Form the multipart email message. - msg = email.mime.multipart.MIMEMultipart('alternative') - msg['Subject'] = subject - msg['From'] = opts.from_address - msg['To'] = to_address - msg.attach(email.mime.text.MIMEText(html_report, "html")) - - # Send the report. - if not opts.dry_run: - s = smtplib.SMTP(opts.host) - s.sendmail(opts.from_address, [to_address], - msg.as_string()) - s.quit() def action_send_run_comparison(name, args): """send a run-vs-run comparison email""" @@ -397,47 +400,47 @@ config = instance.config # Get the database. - db = config.get_database(opts.database) - - # Get the testsuite. - ts = db.testsuite[opts.testsuite] - - # Lookup the two runs. - run_a_id = int(run_a_id) - run_b_id = int(run_b_id) - run_a = ts.query(ts.Run).\ - filter_by(id=run_a_id).first() - run_b = ts.query(ts.Run).\ - filter_by(id=run_b_id).first() - if run_a is None: - parser.error("invalid run ID %r (not in database)" % (run_a_id,)) - if run_b is None: - parser.error("invalid run ID %r (not in database)" % (run_b_id,)) - - # Generate the report. - reports = lnt.server.reporting.runs.generate_run_report( - run_b, baseurl=config.zorgURL, only_html_body=False, result=None, - compare_to=run_a, baseline=None, - aggregation_fn=min) - subject, text_report, html_report, _ = reports - - if opts.subject_prefix is not None: - subject = "%s %s" % (opts.subject_prefix, subject) - - # Form the multipart email message. - msg = email.mime.multipart.MIMEMultipart('alternative') - msg['Subject'] = subject - msg['From'] = opts.from_address - msg['To'] = opts.to_address - msg.attach(email.mime.text.MIMEText(text_report, 'plain')) - msg.attach(email.mime.text.MIMEText(html_report, 'html')) - - # Send the report. - if not opts.dry_run: - s = smtplib.SMTP(opts.host) - s.sendmail(opts.from_address, [opts.to_address], - msg.as_string()) - s.quit() + with contextlib.closing(config.get_database(opts.database)) as db: + + # Get the testsuite. + ts = db.testsuite[opts.testsuite] + + # Lookup the two runs. + run_a_id = int(run_a_id) + run_b_id = int(run_b_id) + run_a = ts.query(ts.Run).\ + filter_by(id=run_a_id).first() + run_b = ts.query(ts.Run).\ + filter_by(id=run_b_id).first() + if run_a is None: + parser.error("invalid run ID %r (not in database)" % (run_a_id,)) + if run_b is None: + parser.error("invalid run ID %r (not in database)" % (run_b_id,)) + + # Generate the report. + reports = lnt.server.reporting.runs.generate_run_report( + run_b, baseurl=config.zorgURL, only_html_body=False, result=None, + compare_to=run_a, baseline=None, + aggregation_fn=min) + subject, text_report, html_report, _ = reports + + if opts.subject_prefix is not None: + subject = "%s %s" % (opts.subject_prefix, subject) + + # Form the multipart email message. + msg = email.mime.multipart.MIMEMultipart('alternative') + msg['Subject'] = subject + msg['From'] = opts.from_address + msg['To'] = opts.to_address + msg.attach(email.mime.text.MIMEText(text_report, 'plain')) + msg.attach(email.mime.text.MIMEText(html_report, 'html')) + + # Send the report. + if not opts.dry_run: + s = smtplib.SMTP(opts.host) + s.sendmail(opts.from_address, [opts.to_address], + msg.as_string()) + s.quit() ### Index: lnt/lnttool/updatedb.py =================================================================== --- lnt/lnttool/updatedb.py +++ lnt/lnttool/updatedb.py @@ -1,5 +1,6 @@ import os from optparse import OptionParser, OptionGroup +import contextlib import lnt.server.instance from lnt.testing.util.commands import note, warning, error, fatal @@ -29,48 +30,50 @@ if opts.testsuite is None: parser.error("--testsuite is required") - + path, = args # Load the instance. instance = lnt.server.instance.Instance.frompath(path) # Get the database and test suite. - db = instance.get_database(opts.database, echo=opts.show_sql) - ts = db.testsuite[opts.testsuite] + with contextlib.closing(instance.get_database(opts.database, + echo=opts.show_sql)) as db: + ts = db.testsuite[opts.testsuite] + + # Compute a list of all the runs to delete. + runs_to_delete = list(opts.delete_runs) + if opts.delete_machines: + runs_to_delete.extend( + id + for id, in ts.query(ts.Run.id).\ + join(ts.Machine).\ + filter(ts.Machine.name.in_(opts.delete_machines))) - # Compute a list of all the runs to delete. - runs_to_delete = list(opts.delete_runs) - if opts.delete_machines: - runs_to_delete.extend( - id - for id, in ts.query(ts.Run.id).\ - join(ts.Machine).\ - filter(ts.Machine.name.in_(opts.delete_machines))) - - # Delete all samples associated with those runs. - ts.query(ts.Sample).\ - filter(ts.Sample.run_id.in_(runs_to_delete)).\ - delete(synchronize_session=False) + # Delete all samples associated with those runs. + ts.query(ts.Sample).\ + filter(ts.Sample.run_id.in_(runs_to_delete)).\ + delete(synchronize_session=False) - # Delete all those runs. - ts.query(ts.Run).\ - filter(ts.Run.id.in_(runs_to_delete)).\ - delete(synchronize_session=False) + # Delete all those runs. + ts.query(ts.Run).\ + filter(ts.Run.id.in_(runs_to_delete)).\ + delete(synchronize_session=False) - # Delete the machines. - for name in opts.delete_machines: - # Delete all FieldChanges associated with this machine. - ids = ts.query(ts.FieldChange.id).\ - join(ts.Machine).filter(ts.Machine.name == name).all() - for i in ids: - ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).delete() + # Delete the machines. + for name in opts.delete_machines: + # Delete all FieldChanges associated with this machine. + ids = ts.query(ts.FieldChange.id).\ + join(ts.Machine).filter(ts.Machine.name == name).all() + for i in ids: + ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).\ + delete() - num_deletes = ts.query(ts.Machine).filter_by(name=name).delete() - if num_deletes == 0: - warning("unable to find machine named: %r" % name) + num_deletes = ts.query(ts.Machine).filter_by(name=name).delete() + if num_deletes == 0: + warning("unable to find machine named: %r" % name) - if opts.commit: - db.commit() - else: - db.rollback() + if opts.commit: + db.commit() + else: + db.rollback() Index: lnt/lnttool/viewcomparison.py =================================================================== --- lnt/lnttool/viewcomparison.py +++ lnt/lnttool/viewcomparison.py @@ -8,6 +8,7 @@ import urllib import webbrowser from optparse import OptionParser, OptionGroup +import contextlib import lnt.util.ImportData from lnt.testing.util.commands import note, warning, error, fatal @@ -86,22 +87,22 @@ lnt.server.db.migrate.update_path(db_path) # Import the two reports. - db = config.get_database('default') - result = lnt.util.ImportData.import_and_report( - config, 'default', db, report_a_path, - '', commit=True) - result = lnt.util.ImportData.import_and_report( - config, 'default', db, report_b_path, - '', commit=True) + with contextlib.closing(config.get_database('default')) as db: + result = lnt.util.ImportData.import_and_report( + config, 'default', db, report_a_path, + '', commit=True) + result = lnt.util.ImportData.import_and_report( + config, 'default', db, report_b_path, + '', commit=True) - # Dispatch another thread to start the webbrowser. - comparison_url = '%s/v4/nts/2?compare_to=1' % (url,) - note("opening comparison view: %s" % (comparison_url,)) - thread.start_new_thread(start_browser, (comparison_url,True)) + # Dispatch another thread to start the webbrowser. + comparison_url = '%s/v4/nts/2?compare_to=1' % (url,) + note("opening comparison view: %s" % (comparison_url,)) + thread.start_new_thread(start_browser, (comparison_url, True)) - # Run the webserver. - app = lnt.server.ui.app.App.create_with_instance(instance) - app.debug = True - app.run(opts.hostname, opts.port, use_reloader=False) + # Run the webserver. + app = lnt.server.ui.app.App.create_with_instance(instance) + app.debug = True + app.run(opts.hostname, opts.port, use_reloader=False) finally: shutil.rmtree(tmpdir) Index: lnt/server/db/v4db.py =================================================================== --- lnt/server/db/v4db.py +++ lnt/server/db/v4db.py @@ -124,6 +124,10 @@ assert (self.real_sample_type and self.status_sample_type), \ "sample types not initialized!" + def close(self): + if self.session is not None: + self.session.close() + @property def testsuite(self): # This is the start of "magic" part of V4DB, which allows us to get Index: lnt/server/ui/app.py =================================================================== --- lnt/server/ui/app.py +++ lnt/server/ui/app.py @@ -85,6 +85,13 @@ return self.testsuite + def close(self): + db = getattr(self, 'db', None) + if db is not None: + db.close() + return super(Request, self).close() + + class App(flask.Flask): @staticmethod def create_with_instance(instance): Index: lnt/util/ImportData.py =================================================================== --- lnt/util/ImportData.py +++ lnt/util/ImportData.py @@ -122,19 +122,19 @@ # Load the shadow database to import into. db_config = config.databases[db_name] shadow_name = db_config.shadow_import - shadow_db = config.get_database(shadow_name) - if shadow_db is None: - raise ValueError,("invalid configuration, shadow import " - "database %r does not exist") % shadow_name - - # Perform the shadow import. - shadow_result = import_and_report(config, shadow_name, - shadow_db, file, format, commit, - show_sample_count, disable_email, - disable_report) - - # Append the shadow result to the result. - result['shadow_result'] = shadow_result + with closing(config.get_database(shadow_name)) as shadow_db: + if shadow_db is None: + raise ValueError, ("invalid configuration, shadow import " + "database %r does not exist") % shadow_name + + # Perform the shadow import. + shadow_result = import_and_report(config, shadow_name, + shadow_db, file, format, commit, + show_sample_count, disable_email, + disable_report) + + # Append the shadow result to the result. + result['shadow_result'] = shadow_result result['success'] = True return result Index: lnt/util/ServerUtil.py =================================================================== --- lnt/util/ServerUtil.py +++ lnt/util/ServerUtil.py @@ -45,11 +45,12 @@ instance = lnt.server.instance.Instance.frompath(path) config = instance.config db_name = 'default' - db = config.get_database(db_name) - if db is None: - raise ValueError("no default database in instance: %r" % (path,)) - return lnt.util.ImportData.import_and_report( - config, db_name, db, file, format='', commit=commit) + with closing(config.get_database(db_name)) as db: + if db is None: + raise ValueError("no default database in instance: %r" % (path,)) + return lnt.util.ImportData.import_and_report( + config, db_name, db, file, format='', commit=commit) + def submitFile(url, file, commit, verbose): # If this is a real url, submit it using urllib.