Index: lnt/trunk/lnt/lnttool/convert.py =================================================================== --- lnt/trunk/lnt/lnttool/convert.py +++ lnt/trunk/lnt/lnttool/convert.py @@ -1,6 +1,10 @@ import os import sys +import click +from lnt import formats + + def convert_data(input, output, inFormat, outFormat): from lnt import formats @@ -13,50 +17,25 @@ out['write'](data, output) output.flush() -def action_convert(name, args): +@click.command("convert") +@click.argument("input", type=click.File('rb'), default="-", required=False) +@click.argument("output", type=click.File('wb'), default="-", required=False) +@click.option("--from", "input_format", show_default=True, + type=click.Choice(formats.format_names + ['']), + default='', help="input format") +@click.option("--to", "output_format", show_default=True, + type=click.Choice(formats.format_names + ['']), + default='plist', help="output format") +def action_convert(input, output, input_format, output_format): """convert between input formats""" - from optparse import OptionParser, OptionGroup - from lnt import formats - parser = OptionParser("%s [options] [, []]" % name) - parser.add_option("", "--from", dest="inputFormat", metavar="NAME", - help="input format name [%default]", default='', - choices=formats.format_names + ['']) - parser.add_option("", "--to", dest="outputFormat", metavar="NAME", - help="output format name [%default]", default='plist', - choices=formats.format_names + ['']) - (opts, args) = parser.parse_args(args) - - input = output = '-' - if len(args) == 0: - pass - elif len(args) == 1: - input, = args - elif len(args) == 2: - input,output = args - else: - parser.error("invalid number of arguments") - - if input == '-': - # Guarantee that we can seek. - import StringIO - data = sys.stdin.read() - inf = StringIO.StringIO(data) - else: - inf = input - - if output == '-': - outf = sys.stdout - else: - outf = open(output, 'wb') - try: try: - convert_data(inf, outf, opts.inputFormat, opts.outputFormat) + convert_data(input, output, input_format, output_format) finally: - if outf != sys.stdout: - outf.close() + if output != sys.stdout: + output.close() except: - if outf != sys.stdout: + if output != sys.stdout: os.remove(output) raise Index: lnt/trunk/lnt/lnttool/create.py =================================================================== --- lnt/trunk/lnt/lnttool/create.py +++ lnt/trunk/lnt/lnttool/create.py @@ -5,9 +5,12 @@ import random import sys -### +import click -kConfigVersion = (0,1,0) +import lnt.testing +import lnt.server.db.migrate + +kConfigVersion = (0, 1, 0) kConfigTemplate = """\ # LNT configuration file # @@ -81,76 +84,58 @@ werkzeug.run_simple('%(hostname)s', 8000, application) """ -### - -import lnt.testing -import lnt.server.db.migrate - -def action_create(name, args): - """create an LLVM nightly test installation""" - - from optparse import OptionParser, OptionGroup - parser = OptionParser("%s [options] " % name) - parser.add_option("", "--name", dest="name", default="LNT", - help="name to use for the installation [%default]") - parser.add_option("", "--config", dest="config", default="lnt.cfg", - help="name of the LNT config file [%default]") - parser.add_option("", "--wsgi", dest="wsgi", default="lnt.wsgi", - help="name of the WSGI app [%default]") - parser.add_option("", "--tmp-dir", dest="tmp_dir", default="lnt_tmp", - help="name of the temp file directory [%default]") - parser.add_option("", "--db-dir", dest="db_dir", default="data", - help="name of the directory to hold databases") - parser.add_option("", "--profile-dir", dest="profile_dir", default="data/profiles", - help="name of the directory to hold databases") - parser.add_option("", "--default-db", dest="default_db", default="lnt.db", - help="name for the default db [%default]", metavar="NAME") - parser.add_option("", "--secret-key", dest="secret_key", default=None, - help="secret key to use for this installation") - parser.add_option("", "--hostname", dest="hostname", - default=platform.uname()[1], - help="host name of the server [%default]", metavar="NAME") - parser.add_option("", "--hostsuffix", dest="hostsuffix", default="perf", - help="suffix at which WSGI app lives [%default]", - metavar="NAME") - parser.add_option("", "--show-sql", dest="show_sql", action="store_true", - help="show SQL statements executed during construction", - default=False) - - (opts, args) = parser.parse_args(args) - if len(args) != 1: - parser.error("invalid number of arguments") - path, = args +@click.command("create", short_help="create an LLVM nightly test installation") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.option("--name", default="LNT", show_default=True, + help="name to use for the installation") +@click.option("--config", default="lnt.cfg", show_default=True, + help="name of the LNT config file") +@click.option("--wsgi", default="lnt.wsgi", show_default=True, + help="name of the WSGI app") +@click.option("--tmp-dir", default="lnt_tmp", show_default=True, + help="name of the temp file directory") +@click.option("--db-dir", default="data", show_default=True, + help="name of the directory to hold databases") +@click.option("--profile-dir", default="data/profiles", show_default=True, + help="name of the directory to hold profiles") +@click.option("--default-db", default="lnt.db", show_default=True, + help="name for the default db") +@click.option("--secret-key", default=None, + help="secret key to use for this installation") +@click.option("--hostname", default=platform.uname()[1], show_default=True, + help="host name of the server") +@click.option("--hostsuffix", default="perf", show_default=True, + help="suffix at which WSGI app lives") +@click.option("--show-sql", is_flag=True, + help="show SQL statements executed during construction") +def action_create(instance_path, name, config, wsgi, tmp_dir, db_dir, + profile_dir, default_db, secret_key, hostname, hostsuffix, + show_sql): + """create an LLVM nightly test installation + +\b +* INSTANCE_PATH should point to a directory that will keep +LNT configuration. + """ # Setup the base LNT logger. logger = logging.getLogger("lnt") logger.setLevel(logging.WARNING) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) + '%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) # Enable full SQL logging, if requested. - if opts.show_sql: + if show_sql: sa_logger = logging.getLogger("sqlalchemy") sa_logger.setLevel(logging.INFO) sa_logger.addHandler(handler) - # Set up locals we use later for substitution. - name = opts.name - config = opts.config - wsgi = opts.wsgi - tmp_dir = opts.tmp_dir - db_dir = opts.db_dir - profile_dir = opts.profile_dir - default_db = opts.default_db - hostname = opts.hostname - hostsuffix = opts.hostsuffix default_db_version = "0.4" - basepath = os.path.abspath(path) + basepath = os.path.abspath(instance_path) if os.path.exists(basepath): raise SystemExit,"error: invalid path: %r already exists" % path @@ -160,10 +145,10 @@ cfg_path = os.path.join(basepath, config) tmp_path = os.path.join(basepath, tmp_dir) wsgi_path = os.path.join(basepath, wsgi) - secret_key = (opts.secret_key or + secret_key = (secret_key or hashlib.sha1(str(random.getrandbits(256))).hexdigest()) - os.mkdir(path) + os.mkdir(instance_path) os.mkdir(tmp_path) # If the path does not contain database type, assume relative path. Index: lnt/trunk/lnt/lnttool/import_data.py =================================================================== --- lnt/trunk/lnt/lnttool/import_data.py +++ lnt/trunk/lnt/lnttool/import_data.py @@ -1,72 +1,61 @@ -import os, pprint, sys, time +import contextlib +import pprint +import sys + +import click import lnt.formats import lnt.util.ImportData import lnt.server.instance -import contextlib -def action_import(name, args): +@click.command("import") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.argument("files", nargs=-1, type=click.Path(exists=True), required=True) +@click.option("--database", default="default", show_default=True, + help="database to modify") +@click.option("--format", "output_format", show_default=True, + type=click.Choice(lnt.formats.format_names + ['']), + default='', help="input format") +@click.option("--commit", type=int, help="commit changes to the database") +@click.option("--show-sql", is_flag=True, help="show SQL statements") +@click.option("--show-sample-count", is_flag=True) +@click.option("--show-raw-result", is_flag=True) +@click.option("--verbose", "-v", is_flag=True, + help="show verbose test results") +@click.option("--quiet", "-q", is_flag=True, help="don't show test results") +@click.option("--no-email", is_flag=True, help="don't send e-mail") +@click.option("--no-report", is_flag=True, help="don't generate report") +def action_import(instance_path, files, database, output_format, commit, + show_sql, show_sample_count, show_raw_result, verbose, + quiet, no_email, no_report): """import test data into a database""" - from optparse import OptionParser, OptionGroup - - parser = OptionParser("%s [options] +"%name) - parser.add_option("", "--database", dest="database", default="default", - help="database to write to [%default]") - parser.add_option("", "--format", dest="format", - choices=lnt.formats.format_names + [''], - default='') - parser.add_option("", "--commit", dest="commit", type=int, - default=False) - parser.add_option("", "--show-sql", dest="show_sql", action="store_true", - default=False) - parser.add_option("", "--show-sample-count", dest="show_sample_count", - action="store_true", default=False) - parser.add_option("", "--show-raw-result", dest="show_raw_result", - action="store_true", default=False) - parser.add_option("-v", "--verbose", dest="verbose", - help="show verbose test results", - action="store_true", default=False) - parser.add_option("-q", "--quiet", dest="quiet", - help="don't show test results", - action="store_true", default=False) - parser.add_option("", "--no-email", dest="no_email", - action="store_true", default=False) - parser.add_option("", "--no-report", dest="no_report", - action="store_true", default=False) - (opts, args) = parser.parse_args(args) - - if len(args) < 2: - parser.error("invalid number of arguments") - - path = args.pop(0) - # Load the LNT instance. - instance = lnt.server.instance.Instance.frompath(path) + instance = lnt.server.instance.Instance.frompath(instance_path) config = instance.config # Get the database. - with contextlib.closing(config.get_database(opts.database, - echo=opts.show_sql)) as db: + with contextlib.closing(config.get_database(database, + echo=show_sql)) as db: # Load the database. success = True - for file in args: + for file_name in files: result = lnt.util.ImportData.import_and_report( - config, opts.database, db, file, - opts.format, opts.commit, opts.show_sample_count, - opts.no_email, opts.no_report) + config, database, db, file_name, + output_format, commit, show_sample_count, + no_email, no_report) success &= result.get('success', False) - if opts.quiet: + if quiet: continue - if opts.show_raw_result: + if show_raw_result: pprint.pprint(result) else: lnt.util.ImportData.print_report_result(result, sys.stdout, sys.stderr, - opts.verbose) + verbose) if not success: raise SystemExit, 1 Index: lnt/trunk/lnt/lnttool/import_report.py =================================================================== --- lnt/trunk/lnt/lnttool/import_report.py +++ lnt/trunk/lnt/lnttool/import_report.py @@ -1,15 +1,27 @@ import os -from optparse import OptionParser + +import click import lnt.testing -def action_importreport(name, args): - """Import simple space separated data into a report to submit.""" - description = """Import simple data into LNT. This takes a space separated +@click.command("importreport", short_help="import simple space separated " + "data into a report to submit.") +@click.argument("input", type=click.File('rb'), default="-", required=False) +@click.argument("output", type=click.File('wb'), default="report.json", + required=False) +@click.option("--testsuite", "suite", default="nts", show_default=True, + required=True, help="short name of the test suite to submit to") +@click.option("--order", required=True, help="Order to submit as number. " + "Ex: a svn revision, or timestamp.") +@click.option("--machine", required=True, + help="the name of the machine to submit under") +def action_importreport(input, output, suite, order, machine): + """Import simple data into LNT. This takes a space separated key value file and creates an LNT report file, which can be submitted to an LNT server. Example input file: + \b foo.exec 123 bar.size 456 foo/bar/baz.size 789 @@ -18,48 +30,21 @@ test suite you are submitting to. to. """ - parser = OptionParser( - "%s [, []] \n\n%s" % (name, description)) - parser.add_option("", "--testsuite", dest="suite", default="nts", - help="Short name of the test suite to submit to." - " [%default]") - parser.add_option("", "--order", dest="order", - help="Order to submit as number. Ex: a svn revision," - " or timestamp.") - parser.add_option("", "--machine", dest="machine", - help="The name of the machine to submit under.") - (opts, args) = parser.parse_args(args) - - input_file_name = None - output = None - - if len(args) == 1: - input_file_name, = args - output = "report.json" - elif len(args) == 2: - input_file_name, output = args - else: - parser.error("Invalid number of arguments.") - if not opts.suite or not opts.order or not opts.machine: - parser.error("Testsuite, order and machine are required.") - - intput_fd = open(input_file_name, 'r') - output_fd = open(output, 'wb') machine_info = {} - run_info = {'tag': opts.suite} - run_info['run_order'] = opts.order - machine = lnt.testing.Machine(opts.machine, + run_info = {'tag': suite} + run_info['run_order'] = order + machine = lnt.testing.Machine(machine, machine_info) - ctime = os.path.getctime(input_file_name) - mtime = os.path.getmtime(input_file_name) + ctime = os.path.getctime(input.name) + mtime = os.path.getmtime(input.name) run = lnt.testing.Run(ctime, mtime, run_info) report = lnt.testing.Report(machine=machine, run=run, tests=[]) - for line in intput_fd.readlines(): + for line in input.readlines(): key, val = line.split() - test = lnt.testing.TestSamples(opts.suite + "." + key, [val]) + test = lnt.testing.TestSamples(suite + "." + key, [val]) report.tests.extend([test]) - output_fd.write(report.render()) + output.write(report.render()) Index: lnt/trunk/lnt/lnttool/main.py =================================================================== --- lnt/trunk/lnt/lnttool/main.py +++ lnt/trunk/lnt/lnttool/main.py @@ -1,110 +1,94 @@ """Implement the command line 'lnt' tool.""" -from convert import action_convert -from create import action_create -from import_data import action_import -from import_report import action_importreport -from lnt import testing -from lnt.testing.util.commands import note, warning, error, fatal, LOGGER_NAME -from optparse import OptionParser, OptionGroup -from updatedb import action_updatedb -from viewcomparison import action_view_comparison -import StringIO -import code -import contextlib -import json -import lnt -import lnt.testing.profile.profile as profile -import lnt.util.ImportData -import lnt.util.multitool import logging import os import sys -import tempfile +import json +import contextlib +import code import werkzeug.contrib.profiler +import click - -def action_runserver(name, args): - """start a new development server""" - - parser = OptionParser("""\ -%s [options] - -Start the LNT server using a development WSGI server. Additional options can -be used to control the server host and port, as well as useful development +import lnt +import lnt.util.ImportData +from lnt.testing.util.commands import note, warning, error, LOGGER_NAME +import lnt.testing.profile.profile as profile +from lnt.tests.nt import NTTest +from lnt.tests.compile import CompileTest +from lnt.tests.test_suite import TestSuiteTest + +from .create import action_create +from .convert import action_convert +from .import_data import action_import +from .updatedb import action_updatedb +from .viewcomparison import action_view_comparison +from .import_report import action_importreport + + +@click.command("runserver", short_help="start a new development server") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.option("--hostname", default="localhost", show_default=True, + help="host interface to use") +@click.option("--port", default=8000, show_default=True, + help="local port to use") +@click.option("--reloader", is_flag=True, help="use WSGI reload monitor") +@click.option("--debugger", is_flag=True, help="use WSGI debugger") +@click.option("--profiler", is_flag=True, help="use WSGI profiler") +@click.option("--profiler-file", help="file to dump profile info to") +@click.option("--profiler-dir", + help="pstat.Stats files are saved to this directory ") +@click.option("--shell", is_flag=True, help="load in shell") +@click.option("--show-sql", is_flag=True, help="show all SQL queries") +@click.option("--threaded", is_flag=True, help="use a threaded server") +@click.option("--processes", default=1, show_default=True, + help="number of processes to use") +def action_runserver(instance_path, hostname, port, reloader, debugger, + profiler, profiler_file, profiler_dir, shell, show_sql, + threaded, processes): + """start a new development server + +\b +Start the LNT server using a development WSGI server. Additional options can be +used to control the server host and port, as well as useful development features such as automatic reloading. The command has built-in support for running the server on an instance which has been packed into a (compressed) tarball. The tarball will be automatically unpacked into a temporary directory and removed on exit. This is useful for passing database instances back and forth, when others only need to be able to -view the results.\ -""" % name) - parser.add_option("", "--hostname", dest="hostname", type=str, - help="host interface to use [%default]", - default='localhost') - parser.add_option("", "--port", dest="port", type=int, metavar="N", - help="local port to use [%default]", default=8000) - parser.add_option("", "--reloader", dest="reloader", default=False, - action="store_true", help="use WSGI reload monitor") - parser.add_option("", "--debugger", dest="debugger", default=False, - action="store_true", help="use WSGI debugger") - parser.add_option("", "--profiler-file", dest="profiler_file", - help="file to dump profile info to [%default]", - default="profiler.log") - parser.add_option("", "--profiler-dir", dest="profiler_dir", - help="pstat.Stats files are saved to this directory " + - "[%default]", - default=None) - parser.add_option("", "--profiler", dest="profiler", default=False, - action="store_true", help="enable WSGI profiler") - parser.add_option("", "--shell", dest="shell", default=False, - action="store_true", help="Load in shell.") - parser.add_option("", "--show-sql", dest="show_sql", default=False, - action="store_true", help="show all SQL queries") - parser.add_option("", "--threaded", dest="threaded", default=False, - action="store_true", help="use a threaded server") - parser.add_option("", "--processes", dest="processes", type=int, - metavar="N", - help="number of processes to use [%default]", default=1) - - (opts, args) = parser.parse_args(args) - if len(args) != 1: - parser.error("invalid number of arguments") - - input_path, = args +view the results. + """ # Setup the base LNT logger. # Root logger in debug. logger = logging.getLogger(LOGGER_NAME) - if opts.debugger: + if debugger: logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) + '%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) # Enable full SQL logging, if requested. - if opts.show_sql: + if show_sql: sa_logger = logging.getLogger("sqlalchemy") - if opts.debugger: + if debugger: sa_logger.setLevel(logging.DEBUG) sa_logger.setLevel(logging.INFO) sa_logger.addHandler(handler) import lnt.server.ui.app - app = lnt.server.ui.app.App.create_standalone(input_path,) - if opts.debugger: + app = lnt.server.ui.app.App.create_standalone(instance_path,) + if debugger: app.debug = True - if opts.profiler: - if opts.profiler_dir: - if not os.path.isdir(opts.profiler_dir): - os.mkdir(opts.profiler_dir) + if profiler: + if profiler_dir: + if not os.path.isdir(profiler_dir): + os.mkdir(profiler_dir) app.wsgi_app = werkzeug.contrib.profiler.ProfilerMiddleware( - app.wsgi_app, stream=open(opts.profiler_file, 'w'), - profile_dir=opts.profiler_dir) - if opts.shell: + app.wsgi_app, stream=open(profiler_file, 'w'), + profile_dir=profiler_dir) + if shell: from flask import current_app from flask import g ctx = app.test_request_context() @@ -115,36 +99,24 @@ shell = code.InteractiveConsole(vars) shell.interact() else: - app.run(opts.hostname, opts.port, - use_reloader=opts.reloader, - use_debugger=opts.debugger, - threaded=opts.threaded, - processes=opts.processes) + app.run(hostname, port, + use_reloader=reloader, + use_debugger=debugger, + threaded=threaded, + processes=processes) -def action_checkformat(name, args): +@click.command("checkformat") +@click.argument("file", "input_file", nargs=-1, type=click.Path(exists=True)) +def action_checkformat(input_file): """check the format of an LNT test report file""" - parser = OptionParser("%s [options] files" % name) - - (opts, args) = parser.parse_args(args) - if len(args) > 1: - parser.error("incorrect number of argments") - - if len(args) == 0: - input = '-' - else: - input, = args - - if input == '-': - input = StringIO.StringIO(sys.stdin.read()) - import lnt.server.db.v4db import lnt.server.config db = lnt.server.db.v4db.V4DB('sqlite:///:memory:', lnt.server.config.Config.dummy_instance()) - result = lnt.util.ImportData.import_and_report(None, None, db, input, - 'json', commit=True) + result = lnt.util.ImportData.import_and_report( + None, None, db, input_file, 'json', commit=True) lnt.util.ImportData.print_report_result(result, sys.stdout, sys.stderr, verbose=True) @@ -160,58 +132,28 @@ print "Results available at: no URL available" -def action_runtest(name, args): +@click.group("runtest", context_settings=dict( + ignore_unknown_options=True, allow_extra_args=True,)) +def action_runtest(): """run a builtin test application""" - - # Runtest accepting options is deprecated, but lets not break the - # world, so collect them anyways and pass them on. - parser = OptionParser("%s test-name [options]" % name) - parser.disable_interspersed_args() - parser.add_option("", "--submit", dest="submit", type=str, default=None) - parser.add_option("", "--commit", dest="commit", type=str, default=None) - parser.add_option("", "--output", dest="output", type=str, default=None) - parser.add_option("-v", "--verbose", dest="verbose", action="store_true") - - (deprecated_opts, args) = parser.parse_args(args) - if len(args) < 1: - parser.error("incorrect number of argments") - - test_name, args = args[0], args[1:] - # Rebuild the deprecated arguments. - for key, val in vars(deprecated_opts).iteritems(): - if val is not None: - if isinstance(val, str): - args.insert(0, val) - args.insert(0, "--" + key) - - warning("--{} should be passed directly to the test suite." - .format(key)) - logger = logging.getLogger(LOGGER_NAME) logger.setLevel(logging.INFO) handler = logging.StreamHandler() handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) + '%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) - import lnt.tests - try: - test_instance = lnt.tests.get_test_instance(test_name) - except KeyError: - parser.error('invalid test name %r' % test_name) - server_results = test_instance.run_test('%s %s' % (name, test_name), args) - _print_result_url(server_results, verbose=True) +action_runtest.add_command(NTTest.cli_wrapper) +action_runtest.add_command(CompileTest.cli_wrapper) +action_runtest.add_command(TestSuiteTest.cli_wrapper) -def action_showtests(name, args): - """show the available built-in tests""" - parser = OptionParser("%s" % name) - (opts, args) = parser.parse_args(args) - if len(args) != 0: - parser.error("incorrect number of argments") +@click.command("showtests") +def action_showtests(): + """show the available built-in tests""" import lnt.tests @@ -223,60 +165,44 @@ lnt.tests.get_test_description(name)) -def action_submit(name, args): +@click.command("submit") +@click.argument("url") +@click.argument("files", nargs=-1, type=click.Path(exists=True), required=True) +@click.option("--commit", show_default=True, type=int, + help="number of days to show in report") +@click.option("--verbose", "-v", is_flag=True, help="show verbose test results") +def action_submit(url, files, commit, verbose): """submit a test report to the server""" - parser = OptionParser("%s [options] +" % name) - parser.add_option("", "--commit", dest="commit", type=int, - help=("whether the result should be committed " - "[%default]"), - default=True) - parser.add_option("-v", "--verbose", dest="verbose", - help="show verbose test results", - action="store_true", default=False) - - (opts, args) = parser.parse_args(args) - if len(args) < 2: - parser.error("incorrect number of argments") - - if not opts.commit: + if not commit: warning("submit called with --commit=0, your results will not be saved" " at the server.") from lnt.util import ServerUtil - files = ServerUtil.submitFiles(args[0], args[1:], - opts.commit, opts.verbose) - for f in files: - if opts.verbose: - lnt.util.ImportData.print_report_result(f, sys.stdout, - sys.stderr, True) - _print_result_url(f, opts.verbose) - - -def action_update(name, args): + files = ServerUtil.submitFiles(url, files, commit, verbose) + for submitted_file in files: + if verbose: + lnt.util.ImportData.print_report_result( + submitted_file, sys.stdout, sys.stderr, True) + _print_result_url(submitted_file, verbose) + +@click.command("update") +@click.argument("db_path") +@click.option("--show-sql", is_flag=True, help="show all SQL queries") +def action_update(db_path, show_sql): """create and or auto-update the given database""" - parser = OptionParser("%s [options] " % name) - parser.add_option("", "--show-sql", dest="show_sql", default=False, - action="store_true", help="show all SQL queries") - - (opts, args) = parser.parse_args(args) - if len(args) != 1: - parser.error("incorrect number of argments") - - db_path, = args - # Setup the base LNT logger. logger = logging.getLogger("lnt") logger.setLevel(logging.INFO) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) + '%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) # Enable full SQL logging, if requested. - if opts.show_sql: + if show_sql: sa_logger = logging.getLogger("sqlalchemy") sa_logger.setLevel(logging.INFO) sa_logger.addHandler(handler) @@ -285,7 +211,28 @@ lnt.server.db.migrate.update_path(db_path) -def action_send_daily_report(name, args): +@click.command("send-daily-report") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.argument("address") +@click.option("--database", default="default", show_default=True, + help="database to use") +@click.option("--testsuite", default="nts", show_default=True, + help="testsuite to use") +@click.option("--host", default="localhost", show_default=True, + help="email relay host to use") +@click.option("--from", "from_address", default=None, required=True, + help="from email address") +@click.option("--today", is_flag=True, + help="send the report for today (instead of most recent)") +@click.option("--subject-prefix", help="add a subject prefix") +@click.option("--dry-run", is_flag=True, help="don't actually send email") +@click.option("--days", default=3, show_default=True, + help="number of days to show in report") +@click.option("--filter-machine-regex", + help="only show machines that contain the regex") +def action_send_daily_report(instance_path, address, database, testsuite, host, + from_address, today, subject_prefix, dry_run, + days, filter_machine_regex): """send a daily report email""" import datetime import email.mime.multipart @@ -294,50 +241,17 @@ import lnt.server.reporting.dailyreport - parser = OptionParser("%s [options]
" % ( - name,)) - parser.add_option("", "--database", dest="database", default="default", - help="database to use [%default]") - parser.add_option("", "--testsuite", dest="testsuite", default="nts", - help="testsuite to use [%default]") - parser.add_option("", "--host", dest="host", default="localhost", - help="email relay host to use [%default]") - parser.add_option("", "--from", dest="from_address", default=None, - help="from email address (required)") - parser.add_option("", "--today", dest="today", action="store_true", - help="send the report for today " + - "(instead of most recent)") - parser.add_option("", "--subject-prefix", dest="subject_prefix", - help="add a subject prefix") - parser.add_option("-n", "--dry-run", dest="dry_run", default=False, - action="store_true", help="Don't actually send email." - " Used for testing.") - parser.add_option("", "--days", dest="days", default=3, type="int", - help="Number of days to show in report.") - parser.add_option("", "--filter-machine-regex", - dest="filter_machine_regex", default=None, - help="only show machines that contain the regex.") - - (opts, args) = parser.parse_args(args) - - if len(args) != 2: - parser.error("invalid number of arguments") - if opts.from_address is None: - parser.error("--from argument is required") - - path, to_address = args - # Load the LNT instance. - instance = lnt.server.instance.Instance.frompath(path) + instance = lnt.server.instance.Instance.frompath(instance_path) config = instance.config # Get the database. - with contextlib.closing(config.get_database(opts.database)) as db: + with contextlib.closing(config.get_database(database)) as db: # Get the testsuite. - ts = db.testsuite[opts.testsuite] + ts = db.testsuite[testsuite] - if opts.today: + if today: date = datetime.datetime.utcnow() else: # Get a timestamp to use to derive the daily report to generate. @@ -357,90 +271,77 @@ report = lnt.server.reporting.dailyreport.DailyReport( ts, year=date.year, month=date.month, day=date.day, day_start_offset_hours=date.hour, for_mail=True, - num_prior_days_to_include=opts.days, - filter_machine_regex=opts.filter_machine_regex) + num_prior_days_to_include=days, + filter_machine_regex=filter_machine_regex) report.build() note("generating HTML report...") ts_url = "%s/db_%s/v4/%s" \ - % (config.zorgURL, opts.database, opts.testsuite) + % (config.zorgURL, database, testsuite) subject = "Daily Report: %04d-%02d-%02d" % ( report.year, report.month, report.day) html_report = report.render(ts_url, only_html_body=False) - if opts.subject_prefix is not None: - subject = "%s %s" % (opts.subject_prefix, subject) + if subject_prefix is not None: + subject = "%s %s" % (subject_prefix, subject) # Form the multipart email message. msg = email.mime.multipart.MIMEMultipart('alternative') msg['Subject'] = subject - msg['From'] = opts.from_address - msg['To'] = to_address + msg['From'] = from_address + msg['To'] = address msg.attach(email.mime.text.MIMEText(html_report, "html")) # Send the report. - if not opts.dry_run: - s = smtplib.SMTP(opts.host) - s.sendmail(opts.from_address, [to_address], + if not dry_run: + s = smtplib.SMTP(host) + s.sendmail(from_address, [address], msg.as_string()) s.quit() -def action_send_run_comparison(name, args): +@click.command("send-run-comparison") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.argument("run_a_id") +@click.argument("run_b_id") +@click.option("--database", default="default", show_default=True, + help="database to use") +@click.option("--testsuite", default="nts", show_default=True, + help="testsuite to use") +@click.option("--host", default="localhost", show_default=True, + help="email relay host to use") +@click.option("--from", "from_address", default=None, required=True, + help="from email address") +@click.option("--to", "to_address", default=None, required=True, + help="to email address") +@click.option("--subject-prefix", help="add a subject prefix") +@click.option("--dry-run", is_flag=True, help="don't actually send email") +def action_send_run_comparison(instance_path, run_a_id, run_b_id, database, + testsuite, host, from_address, to_address, + subject_prefix, dry_run): """send a run-vs-run comparison email""" import email.mime.multipart import email.mime.text import smtplib - import lnt.server.reporting.dailyreport - parser = OptionParser("%s [options] " - " " % (name,)) - parser.add_option("", "--database", dest="database", default="default", - help="database to use [%default]") - parser.add_option("", "--testsuite", dest="testsuite", default="nts", - help="testsuite to use [%default]") - parser.add_option("", "--host", dest="host", default="localhost", - help="email relay host to use [%default]") - parser.add_option("", "--from", dest="from_address", default=None, - help="from email address (required)") - parser.add_option("", "--to", dest="to_address", default=None, - help="to email address (required)") - parser.add_option("", "--subject-prefix", dest="subject_prefix", - help="add a subject prefix") - parser.add_option("-n", "--dry-run", dest="dry_run", default=False, - action="store_true", help="Don't actually send email." - " Used for testing.") - - (opts, args) = parser.parse_args(args) - - if len(args) != 3: - parser.error("invalid number of arguments") - if opts.from_address is None: - parser.error("--from argument is required") - if opts.to_address is None: - parser.error("--to argument is required") - - path, run_a_id, run_b_id = args - # Setup the base LNT logger. logger = logging.getLogger("lnt") logger.setLevel(logging.ERROR) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) + '%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) # Load the LNT instance. - instance = lnt.server.instance.Instance.frompath(path) + instance = lnt.server.instance.Instance.frompath(instance_path) config = instance.config # Get the database. - with contextlib.closing(config.get_database(opts.database)) as db: + with contextlib.closing(config.get_database(database)) as db: # Get the testsuite. - ts = db.testsuite[opts.testsuite] + ts = db.testsuite[testsuite] # Lookup the two runs. run_a_id = int(run_a_id) @@ -450,9 +351,9 @@ run_b = ts.query(ts.Run).\ filter_by(id=run_b_id).first() if run_a is None: - parser.error("invalid run ID %r (not in database)" % (run_a_id,)) + error("invalid run ID %r (not in database)" % (run_a_id,)) if run_b is None: - parser.error("invalid run ID %r (not in database)" % (run_b_id,)) + error("invalid run ID %r (not in database)" % (run_b_id,)) # Generate the report. reports = lnt.server.reporting.runs.generate_run_report( @@ -461,84 +362,69 @@ aggregation_fn=min) subject, text_report, html_report, _ = reports - if opts.subject_prefix is not None: - subject = "%s %s" % (opts.subject_prefix, subject) + if subject_prefix is not None: + subject = "%s %s" % (subject_prefix, subject) # Form the multipart email message. msg = email.mime.multipart.MIMEMultipart('alternative') msg['Subject'] = subject - msg['From'] = opts.from_address - msg['To'] = opts.to_address + msg['From'] = from_address + msg['To'] = to_address msg.attach(email.mime.text.MIMEText(text_report, 'plain')) msg.attach(email.mime.text.MIMEText(html_report, 'html')) # Send the report. - if not opts.dry_run: - s = smtplib.SMTP(opts.host) - s.sendmail(opts.from_address, [opts.to_address], - msg.as_string()) - s.quit() - - -def action_profile(name, args): - if len(args) < 1 or args[0] not in ('upgrade', 'getVersion', - 'getTopLevelCounters', - 'getFunctions', 'getCodeForFunction'): - print >>sys.stderr, """lnt profile - available actions: - upgrade - Upgrade a profile to the latest version - getVersion - Print the version of a profile - getTopLevelCounters - Print the whole-profile counter values - getFunctions - Print an overview of the functions in a profile - getCodeForFunction - Print the code/instruction information for a function -""" - return - - if args[0] == 'upgrade': - parser = OptionParser("lnt profile upgrade ") - opts, args = parser.parse_args(args) - if len(args) < 3: - parser.error('Expected 2 arguments') - - profile.Profile.fromFile(args[1]).upgrade().save(filename=args[2]) - return - - if args[0] == 'getVersion': - parser = OptionParser("lnt profile getVersion ") - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.error('Expected 1 argument') - print profile.Profile.fromFile(args[1]).getVersion() - return - - if args[0] == 'getTopLevelCounters': - parser = OptionParser("lnt profile getTopLevelCounters ") - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.error('Expected 1 argument') - counters = profile.Profile.fromFile(args[1]).getTopLevelCounters() - print json.dumps(counters) - return - - if args[0] == 'getFunctions': - parser = OptionParser("lnt profile getTopLevelCounters ") - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.error('Expected 1 argument') - print json.dumps(profile.Profile.fromFile(args[1]).getFunctions()) - return - - if args[0] == 'getCodeForFunction': - parser = OptionParser("lnt profile getTopLevelCounters ") - opts, args = parser.parse_args(args) - if len(args) < 3: - parser.error('Expected 2 arguments') - code = profile.Profile.fromFile(args[1]).getCodeForFunction(args[2]) - print json.dumps(list(code)) - return - - assert False - -### + if not dry_run: + mail_client = smtplib.SMTP(host) + mail_client.sendmail( + from_address, + [to_address], + msg.as_string()) + mail_client.quit() + + +@click.group("profile") +def action_profile(): + """tools to extract information from profiles""" + return + + +@action_profile.command("upgrade") +@click.argument("input", type=click.Path(exists=True)) +@click.argument("output", type=click.Path(exists=True)) +def command_update(input, output): + """upgrade a profile to the latest version""" + profile.Profile.fromFile(input).upgrade().save(filename=output) + + +@action_profile.command("getVersion") +@click.argument("input", type=click.Path(exists=True)) +def command_get_version(input): + """print the version of a profile""" + print profile.Profile.fromFile(input).getVersion() + + +@action_profile.command("getTopLevelCounters") +@click.argument("input", type=click.Path(exists=True)) +def command_top_level_counters(input): + """print the whole-profile counter values""" + print json.dumps(profile.Profile.fromFile(input).getTopLevelCounters()) + + +@action_profile.command("getFunctions") +@click.argument("input", type=click.Path(exists=True)) +def command_get_functions(input): + """print the functions in a profile""" + print json.dumps(profile.Profile.fromFile(input).getFunctions()) + + +@action_profile.command("getCodeForFunction") +@click.argument("input", type=click.Path(exists=True)) +@click.argument('fn') +def command_code_for_function(input, fn): + """print the code/instruction for a function""" + print json.dumps( + list(profile.Profile.fromFile(input).getCodeForFunction(fn))) def _version_check(): @@ -562,14 +448,44 @@ raise SystemExit("""\ error: installed distribution %s is not current (%s), you may need to reinstall LNT or rerun 'setup.py develop' if using development mode.""" % ( - installed_dist_name, current_dist_name)) + installed_dist_name, current_dist_name)) + +def show_version(ctx, param, value): + """print LNT version""" + if not value or ctx.resilient_parsing: + return + if lnt.__version__: + print "LNT %s" % (lnt.__version__,) + ctx.exit() -tool = lnt.util.multitool.MultiTool(locals(), "LNT %s" % (lnt.__version__,)) +@click.group(invoke_without_command=True, no_args_is_help=True) +@click.option('--version', is_flag=True, callback=show_version, + expose_value=False, is_eager=True, help=show_version.__doc__) +def main(): + """LNT command line tool -def main(*args, **kwargs): +\b +Use ``lnt --help`` for more information on a specific command. + """ _version_check() - return tool.main(*args, **kwargs) + + +main.add_command(action_checkformat) +main.add_command(action_create) +main.add_command(action_convert) +main.add_command(action_import) +main.add_command(action_importreport) +main.add_command(action_profile) +main.add_command(action_runserver) +main.add_command(action_runtest) +main.add_command(action_send_daily_report) +main.add_command(action_send_run_comparison) +main.add_command(action_showtests) +main.add_command(action_submit) +main.add_command(action_update) +main.add_command(action_updatedb) +main.add_command(action_view_comparison) if __name__ == '__main__': main() Index: lnt/trunk/lnt/lnttool/updatedb.py =================================================================== --- lnt/trunk/lnt/lnttool/updatedb.py +++ lnt/trunk/lnt/lnttool/updatedb.py @@ -1,67 +1,60 @@ -import os -from optparse import OptionParser, OptionGroup import contextlib +import click import lnt.server.instance -from lnt.testing.util.commands import note, warning, error, fatal +from lnt.testing.util.commands import warning -def action_updatedb(name, args): - """modify a database""" - - from optparse import OptionParser, OptionGroup - - parser = OptionParser("%s [options] +"%name) - parser.add_option("", "--database", dest="database", default="default", - help="database to modify [%default]") - parser.add_option("", "--testsuite", dest="testsuite", - help="testsuite to modify") - parser.add_option("", "--commit", dest="commit", type=int, - default=False) - parser.add_option("", "--show-sql", dest="show_sql", action="store_true", - default=False) - parser.add_option("", "--delete-machine", dest="delete_machines", - action="append", default=[]) - parser.add_option("", "--delete-run", dest="delete_runs", - action="append", default=[], type=int) - parser.add_option("", "--delete-order", dest="delete_order", default=[], type=int) - (opts, args) = parser.parse_args(args) - - if len(args) != 1: - parser.error("invalid number of arguments") - if opts.testsuite is None: - parser.error("--testsuite is required") - - path, = args +@click.command("updatedb") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.option("--database", default="default", show_default=True, + help="database to modify") +@click.option("--testsuite", required=True, help="testsuite to modify") +@click.option("--tmp-dir", default="lnt_tmp", show_default=True, + help="name of the temp file directory") +@click.option("--commit", type=int, + help="commit changes to the database") +@click.option("--show-sql", is_flag=True, + help="show SQL statements") +@click.option("--delete-machine", "delete_machines", default=[], + type=click.UNPROCESSED, show_default=True, multiple=True, + help="machine names to delete") +@click.option("--delete-run", "delete_runs", default=[], show_default=True, + multiple=True, help="run ids to delete", type=int) +@click.option("--delete-order", default=[], show_default=True, + help="run ids to delete") +def action_updatedb(instance_path, database, testsuite, tmp_dir, commit, + show_sql, delete_machines, delete_runs, delete_order): + """modify a database""" # Load the instance. - instance = lnt.server.instance.Instance.frompath(path) + instance = lnt.server.instance.Instance.frompath(instance_path) # Get the database and test suite. - with contextlib.closing(instance.get_database(opts.database, - echo=opts.show_sql)) as db: - ts = db.testsuite[opts.testsuite] + with contextlib.closing(instance.get_database(database, + echo=show_sql)) as db: + ts = db.testsuite[testsuite] order = None # Compute a list of all the runs to delete. - if opts.delete_order: - order = ts.query(ts.Order).filter(ts.Order.id == opts.delete_order).one() + if delete_order: + order = ts.query(ts.Order).filter(ts.Order.id == delete_order).one() runs_to_delete = ts.query(ts.Run.id).filter(ts.Run.order_id == order.id).all() runs_to_delete = [r[0] for r in runs_to_delete] else: - runs_to_delete = list(opts.delete_runs) - - if opts.delete_machines: + runs_to_delete = list(delete_runs) + + if delete_machines: runs_to_delete.extend( id for id, in ts.query(ts.Run.id).\ join(ts.Machine).\ - filter(ts.Machine.name.in_(opts.delete_machines))) + filter(ts.Machine.name.in_(delete_machines))) # Delete all samples associated with those runs. ts.query(ts.Sample).\ filter(ts.Sample.run_id.in_(runs_to_delete)).\ delete(synchronize_session=False) - + # Delete all FieldChanges and RegressionIndicators for r in runs_to_delete: fcs = ts.query(ts.FieldChange). \ @@ -78,7 +71,7 @@ delete(synchronize_session=False) # Delete the machines. - for name in opts.delete_machines: + for name in delete_machines: # Delete all FieldChanges associated with this machine. ids = ts.query(ts.FieldChange.id).\ join(ts.Machine).filter(ts.Machine.name == name).all() @@ -92,7 +85,7 @@ if order: ts.delete(order) - if opts.commit: + if commit: db.commit() else: db.rollback() Index: lnt/trunk/lnt/lnttool/viewcomparison.py =================================================================== --- lnt/trunk/lnt/lnttool/viewcomparison.py +++ lnt/trunk/lnt/lnttool/viewcomparison.py @@ -7,9 +7,10 @@ import time import urllib import webbrowser -from optparse import OptionParser import contextlib +import click + from lnt.util.ImportData import import_and_report from lnt.testing.util.commands import note, warning @@ -35,35 +36,28 @@ time.sleep(.01) else: warning('unable to detect that server started') - + if debug: note('opening webbrowser...') webbrowser.open(url) -def action_view_comparison(name, args): +@click.command("view-comparison") +@click.argument("report_a", type=click.Path(exists=True)) +@click.argument("report_b", type=click.Path(exists=True)) +@click.option("--hostname", default="localhost", show_default=True, + help="host interface to use") +@click.option("--port", default=8000, show_default=True, + help="local port to use") +@click.option("--dry-run", is_flag=True, + help="do a dry run through the comparison") +def action_view_comparison(report_a, report_b, hostname, port, dry_run): """view a report comparison using a temporary server""" import lnt.server.instance import lnt.server.ui.app import lnt.server.db.migrate - parser = OptionParser("%s [options] " % (name,)) - parser.add_option("", "--hostname", dest="hostname", type=str, - help="host interface to use [%default]", - default='localhost') - parser.add_option("", "--port", dest="port", type=int, metavar="N", - help="local port to use [%default]", default=8000) - parser.add_option("", "--dry-run", dest="dry_run", - help="Do a dry run through the comparison. [%default]" - " [%default]", action="store_true", default=False) - (opts, args) = parser.parse_args(args) - - if len(args) != 2: - parser.error("invalid number of arguments") - - report_a_path, report_b_path = args - # Set up the default logger. logger = logging.getLogger("lnt") logger.setLevel(logging.ERROR) @@ -78,7 +72,7 @@ try: # Create a temporary instance. - url = 'http://%s:%d' % (opts.hostname, opts.port) + url = 'http://%s:%d' % (hostname, port) db_path = os.path.join(tmpdir, 'data.db') db_info = lnt.server.config.DBInfo( 'sqlite:///%s' % (db_path,), '0.4', None, @@ -86,7 +80,8 @@ # _(self, name, zorgURL, dbDir, tempDir, # profileDir, secretKey, databases, blacklist): config = lnt.server.config.Config('LNT', url, db_path, tmpdir, - None, "Not secret key.", {'default': db_info}, + None, "Not secret key.", + {'default': db_info}, None) instance = lnt.server.instance.Instance(None, config) @@ -95,25 +90,23 @@ # Import the two reports. with contextlib.closing(config.get_database('default')) as db: + r = import_and_report( + config, 'default', db, report_a, '', commit=True) import_and_report( - config, 'default', db, report_a_path, - '', commit=True) - import_and_report( - config, 'default', db, report_b_path, - '', commit=True) + config, 'default', db, report_b, '', commit=True) # Dispatch another thread to start the webbrowser. comparison_url = '%s/v4/nts/2?compare_to=1' % (url,) note("opening comparison view: %s" % (comparison_url,)) - - if not opts.dry_run: + + if not dry_run: thread.start_new_thread(start_browser, (comparison_url, True)) # Run the webserver. app = lnt.server.ui.app.App.create_with_instance(instance) app.debug = True - - if opts.dry_run: + + if dry_run: # Don't catch out exceptions. app.testing = True # Create a test client. @@ -121,6 +114,6 @@ response = client.get(comparison_url) assert response.status_code == 200, "Page did not return 200." else: - app.run(opts.hostname, opts.port, use_reloader=False) + app.run(hostname, port, use_reloader=False) finally: shutil.rmtree(tmpdir) Index: lnt/trunk/lnt/tests/__init__.py =================================================================== --- lnt/trunk/lnt/tests/__init__.py +++ lnt/trunk/lnt/tests/__init__.py @@ -23,9 +23,9 @@ # Allow hyphens instead of underscores when specifying the test on the command # line. (test-suite instead of test_suite). name = name.replace('-', '_') - + if name not in known_tests: - raise KeyError,name + raise KeyError, name module = getattr(__import__('lnt.tests.%s' % name, level=0).tests, name) Index: lnt/trunk/lnt/tests/builtintest.py =================================================================== --- lnt/trunk/lnt/tests/builtintest.py +++ lnt/trunk/lnt/tests/builtintest.py @@ -11,20 +11,30 @@ import lnt.util.ImportData as ImportData import lnt.server.config as server_config import lnt.server.db.v4db -import lnt.server.config + + +class OptsContainer(object): + pass class BuiltinTest(object): def __init__(self): + self.opts = OptsContainer() pass + def _fatal(self, msg): + """This simulate the output provided by OptionParser.error""" + prog_name = os.path.basename(sys.argv[0]) + sys.stderr.write("%s error: %s\n" % (prog_name, msg)) + sys.exit(2) + def describe(self): """"describe() -> str Return a short description of the test. """ - def run_test(self, name, args): + def run_test(self, opts): """run_test(name, args) -> lnt.testing.Report Execute the test (accessed via name, for use in the usage message) with @@ -83,3 +93,11 @@ ImportData.print_report_result(server_report, sys.stdout, sys.stderr, config.verbose) return server_report + + @staticmethod + def show_results_url(server_results): + """Print the result URL""" + if server_results.get('result_url'): + print "Results available at:", server_results['result_url'] + else: + print "Results available at: no URL available" Index: lnt/trunk/lnt/tests/compile.py =================================================================== --- lnt/trunk/lnt/tests/compile.py +++ lnt/trunk/lnt/tests/compile.py @@ -13,14 +13,14 @@ from datetime import datetime import collections -import builtintest -from optparse import OptionParser, OptionGroup +import click import lnt.testing import lnt.testing.util.compilers +from lnt.testing.util import commands, machineinfo from lnt.testing.util.commands import note, fatal, resolve_command_path from lnt.testing.util.misc import timestamp -from lnt.testing.util import commands, machineinfo +from lnt.tests import builtintest from lnt.util import stats @@ -34,6 +34,8 @@ ('-Oz',), ('-Oz', '-g')] +opts = None + def args_to_quoted_string(args): def quote_arg(arg): @@ -704,125 +706,125 @@ def describe(self): return 'Single file compile-time performance testing' - def run_test(self, name, args): + # FIXME: an equivalent to argparse's add_argument_group is not implemented + # on click. Need to review it when such functionality is available. + # https://github.com/pallets/click/issues/373 + @staticmethod + @click.command("compile", help=usage_info, + short_help="Single file compile-time performance testing") + @click.argument("label", default=platform.uname()[1], required=False, + type=click.UNPROCESSED) + @click.option("-s", "--sandbox", "sandbox_path", required=True, + help="Parent directory to build and run tests in", + type=click.UNPROCESSED, default=None, metavar="PATH") + # Test Options + @click.option("--no-timestamp", "timestamp_build", + help="Don't timestamp build directory (for testing)", + flag_value=False, default=True) + @click.option("--cc", "cc", type=click.UNPROCESSED, required=True, + help="Path to the compiler under test") + @click.option("--cxx", "cxx", + help="Path to the C++ compiler to test", + type=click.UNPROCESSED, default=None) + @click.option("--ld", "ld", + help="Path to the c linker to use. (Xcode Distinction)", + type=click.UNPROCESSED, default=None) + @click.option("--ldxx", "ldxx", + help="Path to the cxx linker to use. (Xcode Distinction)", + type=click.UNPROCESSED, default=None) + @click.option("--runn", "runn", + help="Path to runN tool.", + type=click.UNPROCESSED, default="runN") + @click.option("--test-externals", "test_suite_externals", required=True, + help="Path to the LLVM test-suite externals", + type=click.UNPROCESSED, default=None, metavar="PATH") + @click.option("--machine-param", "machine_parameters", + metavar="NAME=VAL", + help="Add 'NAME' = 'VAL' to the machine parameters", + type=click.UNPROCESSED, multiple=True, default=[]) + @click.option("--run-param", "run_parameters", + metavar="NAME=VAL", + help="Add 'NAME' = 'VAL' to the run parameters", + type=click.UNPROCESSED, multiple=True, default=[]) + @click.option("--run-order", "run_order", metavar="STR", + help="String to use to identify and order this run", + type=click.UNPROCESSED, default=None) + @click.option("--test-subdir", "test_subdir", + help="Subdirectory of test external dir to look for " + "tests in.", + type=click.UNPROCESSED, default="lnt-compile-suite-src") + # Test Selection + @click.option("--no-memory-profiling", "memory_profiling", + help="Disable memory profiling", + flag_value=False, default=True) + @click.option("--multisample", "run_count", metavar="N", + help="Accumulate test data from multiple runs", + type=int, default=3) + @click.option("--min-sample-time", "min_sample_time", + help="Ensure all tests run for at least N seconds", + metavar="N", type=float, default=.5) + @click.option("--save-temps", "save_temps", + help="Save temporary build output files", is_flag=True) + @click.option("--show-tests", "show_tests", + help="Only list the availables tests that will be run", + is_flag=True) + @click.option("--test", "tests", metavar="NAME", + help="Individual test to run", + multiple=True, default=[], type=click.UNPROCESSED) + @click.option("--test-filter", "test_filters", + help="Run tests matching the given pattern", + metavar="REGEXP", multiple=True, default=[], + type=click.UNPROCESSED) + @click.option("--flags-to-test", "flags_to_test", + help="Add a set of flags to test (space separated)", + metavar="FLAGLIST", multiple=True, default=[], + type=click.UNPROCESSED) + @click.option("--jobs-to-test", "jobs_to_test", + help="Add a job count to test (full builds)", + metavar="NUM", multiple=True, default=[], type=int) + @click.option("--config-to-test", "configs_to_test", + help="Add build configuration to test (full builds)", + metavar="NAME", multiple=True, default=[], + type=click.Choice(['Debug', 'Release'])) + # Output Options + @click.option("--no-machdep-info", "use_machdep_info", + help=("Don't put machine (instance) dependent " + "variables in machine info"), + flag_value=False, default=True) + @click.option("--machine-name", "machine_name", type=click.UNPROCESSED, + help="Machine name to use in submission", + default=platform.uname()[1]) + @click.option("--submit", "submit_url", metavar="URLORPATH", + help=("autosubmit the test result to the given server " + "(or local instance)"), + type=click.UNPROCESSED, default=None) + @click.option("--commit", "commit", + help="whether the autosubmit result should be committed", + type=int, default=True) + @click.option("--output", "output", metavar="PATH", + help="write raw report data to PATH (or stdout if '-')") + @click.option("-v", "--verbose", "verbose", + help="show verbose test results", is_flag=True) + def cli_wrapper(*args, **kwargs): + """Single file compile-time performance testing""" global opts - parser = OptionParser( - ("%(name)s [options] []\n" + - usage_info) % locals()) - parser.add_option("-s", "--sandbox", dest="sandbox_path", - help="Parent directory to build and run tests in", - type=str, default=None, metavar="PATH") - - group = OptionGroup(parser, "Test Options") - group.add_option("", "--no-timestamp", dest="timestamp_build", - help="Don't timestamp build directory (for testing)", - action="store_false", default=True) - group.add_option("", "--cc", dest="cc", type='str', - help="Path to the compiler under test", - action="store", default=None) - group.add_option("", "--cxx", dest="cxx", - help="Path to the C++ compiler to test", - type=str, default=None) - group.add_option("", "--ld", dest="ld", - help="Path to the c linker to use. (Xcode Distinction)", - type=str, default=None) - group.add_option("", "--ldxx", dest="ldxx", - help="Path to the cxx linker to use. (Xcode Distinction)", - type=str, default=None) - group.add_option("", "--runn", dest="runn", - help="Path to runN tool.", - type=str, default="runN") - group.add_option("", "--test-externals", dest="test_suite_externals", - help="Path to the LLVM test-suite externals", - type=str, default=None, metavar="PATH") - group.add_option("", "--machine-param", dest="machine_parameters", - metavar="NAME=VAL", - help="Add 'NAME' = 'VAL' to the machine parameters", - type=str, action="append", default=[]) - group.add_option("", "--run-param", dest="run_parameters", - metavar="NAME=VAL", - help="Add 'NAME' = 'VAL' to the run parameters", - type=str, action="append", default=[]) - group.add_option("", "--run-order", dest="run_order", metavar="STR", - help="String to use to identify and order this run", - action="store", type=str, default=None) - group.add_option("", "--test-subdir", dest="test_subdir", - help="Subdirectory of test external dir to look for tests in.", - type=str, default="lnt-compile-suite-src") - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Selection") - group.add_option("", "--no-memory-profiling", dest="memory_profiling", - help="Disable memory profiling", - action="store_false", default=True) - group.add_option("", "--multisample", dest="run_count", metavar="N", - help="Accumulate test data from multiple runs", - action="store", type=int, default=3) - group.add_option("", "--min-sample-time", dest="min_sample_time", - help="Ensure all tests run for at least N seconds", - metavar="N", action="store", type=float, default=.5) - group.add_option("", "--save-temps", dest="save_temps", - help="Save temporary build output files", - action="store_true", default=False) - group.add_option("", "--show-tests", dest="show_tests", - help="Only list the availables tests that will be run", - action="store_true", default=False) - group.add_option("", "--test", dest="tests", metavar="NAME", - help="Individual test to run", - action="append", default=[]) - group.add_option("", "--test-filter", dest="test_filters", - help="Run tests matching the given pattern", - metavar="REGEXP", action="append", default=[]) - group.add_option("", "--flags-to-test", dest="flags_to_test", - help="Add a set of flags to test (space separated)", - metavar="FLAGLIST", action="append", default=[]) - group.add_option("", "--jobs-to-test", dest="jobs_to_test", - help="Add a job count to test (full builds)", - metavar="NUM", action="append", default=[], type=int) - group.add_option("", "--config-to-test", dest="configs_to_test", - help="Add build configuration to test (full builds)", - metavar="NAME", action="append", default=[], - choices=('Debug', 'Release')) - parser.add_option_group(group) - - group = OptionGroup(parser, "Output Options") - group.add_option("", "--no-machdep-info", dest="use_machdep_info", - help=("Don't put machine (instance) dependent " - "variables in machine info"), - action="store_false", default=True) - group.add_option("", "--machine-name", dest="machine_name", type='str', - help="Machine name to use in submission [%default]", - action="store", default=platform.uname()[1]) - group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH", - help=("autosubmit the test result to the given server " - "(or local instance) [%default]"), - type=str, default=None) - group.add_option("", "--commit", dest="commit", - help=("whether the autosubmit result should be committed " - "[%default]"), - type=int, default=True) - group.add_option("", "--output", dest="output", metavar="PATH", - help="write raw report data to PATH (or stdout if '-')", - action="store", default=None) - group.add_option("-v", "--verbose", dest="verbose", - help="show verbose test results", - action="store_true", default=False) - - parser.add_option_group(group) - opts, args = parser.parse_args(args) + compile_test = CompileTest() + opts = compile_test.opts + + for key, value in kwargs.items(): + setattr(compile_test.opts, key, value) - if len(args) != 0: - parser.error("invalid number of arguments") + results = compile_test.run_test(compile_test.opts) + compile_test.show_results_url(results) - if opts.cc is None: - parser.error("You must specify a --cc argument.") + def run_test(self, opts): # Resolve the cc_under_test path. opts.cc = resolve_command_path(opts.cc) if not lnt.testing.util.compilers.is_valid(opts.cc): - parser.error('--cc does not point to a valid executable.') + self._fatal('--cc does not point to a valid executable.') # Attempt to infer the cxx compiler if not given. if opts.cc and opts.cxx is None: @@ -830,26 +832,20 @@ if opts.cxx is not None: note("inferred C++ compiler under test as: %r" % (opts.cxx,)) - # Validate options. - if opts.cc is None: - parser.error('--cc is required') if opts.cxx is None: - parser.error('--cxx is required (and could not be inferred)') - if opts.sandbox_path is None: - parser.error('--sandbox is required') - if opts.test_suite_externals is None: - parser.error("--test-externals option is required") + self._fatal('--cxx is required (and could not be inferred)') + # Force the CC and CXX variables to be absolute paths. cc_abs = os.path.abspath(commands.which(opts.cc)) cxx_abs = os.path.abspath(commands.which(opts.cxx)) if not os.path.exists(cc_abs): - parser.error("unable to determine absolute path for --cc: %r" % ( - opts.cc,)) + self._fatal("unable to determine absolute path for --cc: %r" % ( + opts.cc,)) if not os.path.exists(cxx_abs): - parser.error("unable to determine absolute path for --cc: %r" % ( - opts.cc,)) + self._fatal("unable to determine absolute path for --cc: %r" % ( + opts.cc,)) opts.cc = cc_abs opts.cxx = cxx_abs @@ -878,8 +874,8 @@ os.mkdir(g_output_dir) except OSError as e: if e.errno == errno.EEXIST: - parser.error("sandbox output directory %r already exists!" % ( - g_output_dir,)) + self._fatal("sandbox output directory %r already exists!" % ( + g_output_dir,)) else: raise @@ -1003,9 +999,9 @@ requested_tests = set(opts.tests) missing_tests = requested_tests - all_test_names if missing_tests: - parser.error(("invalid test names %s, use --show-tests to " - "see available tests") % - (", ".join(map(repr, missing_tests)), )) + self._fatal(("invalid test names %s, use --show-tests to " + "see available tests") % + (", ".join(map(repr, missing_tests)), )) # Validate the test filters. test_filters = [re.compile(pattern) @@ -1019,7 +1015,7 @@ for filter in test_filters if filter.search(test[0])])] if not tests_to_run: - parser.error( + self._fatal( "no tests requested (invalid --test or --test-filter options)!") # Ensure output directory is available. @@ -1051,11 +1047,10 @@ test_name = '%s.%s' % (tag, name) if not success: testsamples.append(lnt.testing.TestSamples( - test_name + '.status', - [lnt.testing.FAIL])) + test_name + '.status', [lnt.testing.FAIL])) if samples: testsamples.append(lnt.testing.TestSamples( - test_name, samples)) + test_name, samples)) end_time = datetime.utcnow() g_log.info('run complete') @@ -1078,9 +1073,3 @@ server_report = self.submit(lnt_report_path, opts) return server_report - - -def create_instance(): - return CompileTest() - -__all__ = ['create_instance'] Index: lnt/trunk/lnt/tests/nt.py =================================================================== --- lnt/trunk/lnt/tests/nt.py +++ lnt/trunk/lnt/tests/nt.py @@ -8,13 +8,13 @@ import glob import time import traceback -from datetime import datetime -from optparse import OptionParser, OptionGroup import urllib2 import shlex import pipes import resource +import click + import lnt.testing import lnt.testing.util.compilers import lnt.util.ImportData as ImportData @@ -206,7 +206,7 @@ def qemu_user_mode_command(self): """ The command used for qemu user mode """ assert self.qemu_user_mode - qemu_cmd_line = [self.qemu_user_mode] + self.qemu_flags + qemu_cmd_line = [self.qemu_user_mode] + list(self.qemu_flags) if self.qemu_string: qemu_cmd_line += _unix_quote_args(self.qemu_string) return ' '.join(qemu_cmd_line) @@ -293,10 +293,10 @@ while build_mode: for (name, key) in (('+Asserts', 'ENABLE_ASSERTIONS'), - ('+Checks', 'ENABLE_EXPENSIVE_CHECKS'), - ('+Coverage', 'ENABLE_COVERAGE'), - ('+Debug', 'DEBUG_SYMBOLS'), - ('+Profile', 'ENABLE_PROFILING')): + ('+Checks', 'ENABLE_EXPENSIVE_CHECKS'), + ('+Coverage', 'ENABLE_COVERAGE'), + ('+Debug', 'DEBUG_SYMBOLS'), + ('+Profile', 'ENABLE_PROFILING')): if build_mode.startswith(name): build_mode = build_mode[len(name):] make_variables[key] = '1' @@ -1372,7 +1372,7 @@ test_name = '.'.join(fields[1:test_type_size]) updating_entry = collated_results.get(test_name, - PastRunData(test_name)) + PastRunData(test_name)) # Filter out "LNTBased" benchmarks for rerun, they # won't work. LNTbased look like nts.module.test @@ -1404,7 +1404,7 @@ test_name = '.'.join(fields[:-1]) test_type = fields[-1] - new_entry = collated_results.get(test_name, None) + new_entry = collated_results.get(test_name, None) # Some tests will come from the server, which we did not run locally. # Drop them. if new_entry is None: @@ -1459,6 +1459,7 @@ usage_info = """ + Script for running the tests in LLVM's test-suite repository. This script expects to run against a particular LLVM source tree, build, and @@ -1467,7 +1468,8 @@ Basic usage: - %(name)s \\ +\b + lnt runtest nt \\ --sandbox FOO \\ --cc ~/llvm.obj.64/Release/bin/clang \\ --test-suite ~/llvm-test-suite @@ -1496,288 +1498,241 @@ def describe(self): return 'LLVM test-suite compile and execution tests' - def run_test(self, name, args): - parser = OptionParser( - ("%(name)s [options] tester-name\n" + usage_info) % locals()) - - group = OptionGroup(parser, "Sandbox Options") - group.add_option("-s", "--sandbox", dest="sandbox_path", - help="Parent directory to build and run tests in", - type=str, default=None, metavar="PATH") - group.add_option("", "--no-timestamp", dest="timestamp_build", - help="Don't timestamp build directory (for testing)", - action="store_false", default=True) - group.add_option("", "--no-configure", dest="run_configure", - help=("Don't run configure if Makefile.config is " - "present (only useful with --no-timestamp)"), - action="store_false", default=True) - parser.add_option_group(group) - - group = OptionGroup(parser, "Inputs") - group.add_option("", "--without-llvm", dest="without_llvm", - help="Don't use any LLVM source or build products", - action="store_true", default=False) - group.add_option("", "--llvm-src", dest="llvm_src_root", - help="Path to the LLVM source tree", - type=str, default=None, metavar="PATH") - group.add_option("", "--llvm-obj", dest="llvm_obj_root", - help="Path to the LLVM source tree", - type=str, default=None, metavar="PATH") - group.add_option("", "--test-suite", dest="test_suite_root", - help="Path to the LLVM test-suite sources", - type=str, default=None, metavar="PATH") - group.add_option("", "--test-externals", dest="test_suite_externals", - help="Path to the LLVM test-suite externals", - type=str, default='/dev/null', metavar="PATH") - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Compiler") - group.add_option("", "--cc", dest="cc_under_test", metavar="CC", - help="Path to the C compiler to test", - type=str, default=None) - group.add_option("", "--cxx", dest="cxx_under_test", metavar="CXX", - help="Path to the C++ compiler to test", - type=str, default=None) - group.add_option("", "--cc-reference", dest="cc_reference", - help="Path to the reference C compiler", - type=str, default=None) - group.add_option("", "--cxx-reference", dest="cxx_reference", - help="Path to the reference C++ compiler", - type=str, default=None) - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Options") - group.add_option("", "--arch", dest="arch", - help="Set -arch in TARGET_FLAGS [%default]", - type=str, default=None) - group.add_option("", "--llvm-arch", dest="llvm_arch", - help="Set the ARCH value used in the makefiles to " - "[%default]", - type=str, default=None) - group.add_option("", "--make-param", dest="make_parameters", - metavar="NAME=VAL", - help="Add 'NAME' = 'VAL' to the makefile parameters", - type=str, action="append", default=[]) - group.add_option("", "--isysroot", dest="isysroot", metavar="PATH", - help="Set -isysroot in TARGET_FLAGS [%default]", - type=str, default=None) - group.add_option("", "--liblto-path", dest="liblto_path", - metavar="PATH", - help=("Specify the path to the libLTO library " - "[%default]"), - type=str, default=None) - - group.add_option("", "--mcpu", dest="mcpu", - help="Set -mcpu in TARGET_LLCFLAGS [%default]", - type=str, default=None, metavar="CPU") - group.add_option("", "--relocation-model", dest="relocation_model", - help=("Set -relocation-model in TARGET_LLCFLAGS " - "[%default]"), - type=str, default=None, metavar="MODEL") - group.add_option("", "--disable-fp-elim", dest="disable_fp_elim", - help=("Set -disable-fp-elim in TARGET_LLCFLAGS"), - action="store_true", default=False) - - group.add_option("", "--optimize-option", dest="optimize_option", - help="Set optimization level for {LLC_,LLI_,}OPTFLAGS", - choices=('-O0', '-O1', '-O2', '-O3', '-Os', '-Oz'), - default='-O3') - group.add_option("", "--cflag", dest="cflags", - help="Additional flags to set in TARGET_FLAGS", - action="append", type=str, default=[], metavar="FLAG") - group.add_option("", "--cflags", dest="cflag_string", - help="Additional flags to set in TARGET_FLAGS, space separated string. " - "These flags are appended after *all* the individual --cflag arguments.", - type=str, default='', metavar="FLAG") - group.add_option("", "--mllvm", dest="mllvm", - help="Add -mllvm FLAG to TARGET_FLAGS", - action="append", type=str, default=[], metavar="FLAG") - group.add_option("", "--spec-with-pgo", help="Use PGO with SPEC", - action="store_true") - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Selection") - group.add_option("", "--build-mode", dest="build_mode", metavar="NAME", - help="Select the LLVM build mode to use [%default]", - type=str, action="store", default='Release+Asserts') - - group.add_option("", "--simple", dest="test_simple", - help="Use TEST=simple instead of TEST=nightly", - action="store_true", default=False) - group.add_option("", "--test-style", dest="test_style", - help="Set the test style to run [%default]", - choices=('nightly', 'simple'), default='simple') - - group.add_option("", "--test-time-stat", dest="test_time_stat", - help="Set the test timing statistic to gather " - "[%default]", - choices=('user', 'real'), default='user') - - group.add_option("", "--disable-cxx", dest="test_cxx", - help="Disable C++ tests", - action="store_false", default=True) - - group.add_option("", "--disable-externals", dest="test_externals", - help="Disable test suite externals (if configured)", - action="store_false", default=True) - group.add_option("", "--enable-integrated-as", - dest="test_integrated_as", - help="Enable TEST_INTEGRATED_AS tests", - action="store_true", default=False) - group.add_option("", "--enable-jit", dest="test_jit", - help="Enable JIT tests", - action="store_true", default=False) - group.add_option("", "--disable-llc", dest="test_llc", - help="Disable LLC tests", - action="store_false", default=True) - group.add_option("", "--enable-llcbeta", dest="test_llcbeta", - help="Enable LLCBETA tests", - action="store_true", default=False) - group.add_option("", "--disable-lto", dest="test_lto", - help="Disable use of link-time optimization", - action="store_false", default=True) - - group.add_option("", "--small", dest="test_small", - help="Use smaller test inputs and disable large tests", - action="store_true", default=False) - group.add_option("", "--large", dest="test_large", - help="Use larger test inputs", - action="store_true", default=False) - group.add_option("", "--spec-with-ref", dest="test_spec_ref", - help="Use reference test inputs for SPEC. " - "This is currently experimental", - action="store_true", default=False) - group.add_option("", "--benchmarking-only", dest="test_benchmarking_only", - help="Benchmarking-only mode", - action="store_true", default=False) - - group.add_option("", "--only-test", dest="only_test", metavar="PATH", - help="Only run tests under PATH", - type=str, default=None) - group.add_option("", "--include-test-examples", - dest="include_test_examples", - help="Include test module examples [%default]", - action="store_true", default=False) - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Execution") - group.add_option("-j", "--threads", dest="threads", - help="Number of testing threads", - type=int, default=1, metavar="N") - group.add_option("", "--build-threads", dest="build_threads", - help="Number of compilation threads", - type=int, default=0, metavar="N") - group.add_option("", "--use-perf", dest="use_perf", - help=("Use perf to obtain high accuracy timing" - "[%default]"), - type=str, default=None) - group.add_option("", "--rerun", dest="rerun", - help="Rerun tests that have regressed.", - action="store_true", default=False) - group.add_option("", "--remote", dest="remote", - help=("Execute remotely, see " - "--remote-{host,port,user,client} [%default]"), - action="store_true", default=False) - group.add_option("", "--remote-host", dest="remote_host", - help="Set remote execution host [%default]", - type=str, default="localhost", metavar="HOST") - group.add_option("", "--remote-port", dest="remote_port", - help="Set remote execution port [%default] ", - type=int, default=None, metavar="PORT",) - group.add_option("", "--remote-user", dest="remote_user", - help="Set remote execution user [%default]", - type=str, default=None, metavar="USER",) - group.add_option("", "--remote-client", dest="remote_client", - help="Set remote execution client [%default]", - type=str, default="ssh", metavar="RSH",) - - group.add_option("", "--use-ios-simulator", dest="ios_simulator_sdk", - help=("Execute using an iOS simulator SDK (using " - "environment overrides)"), - type=str, default=None, metavar="SDKPATH") - group.add_option("", "--use-isolation", dest="use_isolation", - help=("Execute using a sandboxing profile to limit " - "OS access (e.g., to the network or " - "non-test directories)"), - action="store_true", default=False) - - group.add_option("", "--qemu-user-mode", dest="qemu_user_mode", - help=("Enable qemu user mode emulation using this " - "qemu executable [%default]"), - type=str, default=None) - group.add_option("", "--qemu-flag", dest="qemu_flags", - help="Additional flags to pass to qemu", - action="append", type=str, default=[], metavar="FLAG") - group.add_option("", "--qemu-flags", dest="qemu_string", - help="Additional flags to pass to qemu, space separated string. " - "These flags are appended after *all* the individual " - "--qemu-flag arguments.", - type=str, default='', metavar="FLAG") - - group.add_option("", "--multisample", dest="multisample", - help="Accumulate test data from multiple runs", - type=int, default=None, metavar="N") - parser.add_option_group(group) - - group = OptionGroup(parser, "Output Options") - group.add_option("", "--no-auto-name", dest="auto_name", - help="Don't automatically derive submission name", - action="store_false", default=True) - group.add_option("", "--no-machdep-info", dest="use_machdep_info", - help=("Don't put machine (instance) dependent " - "variables with machine info"), - action="store_false", default=True) - group.add_option("", "--run-order", dest="run_order", metavar="STR", - help="String to use to identify and order this run", - action="store", type=str, default=None) - group.add_option("", "--machine-param", dest="machine_parameters", - metavar="NAME=VAL", - help="Add 'NAME' = 'VAL' to the machine parameters", - type=str, action="append", default=[]) - group.add_option("", "--run-param", dest="run_parameters", - metavar="NAME=VAL", - help="Add 'NAME' = 'VAL' to the run parameters", - type=str, action="append", default=[]) - group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH", - help=("autosubmit the test result to the given server" - " (or local instance) [%default]"), - type=str, default=[], action="append") - group.add_option("", "--commit", dest="commit", - help=("whether the autosubmit result should be committed " - "[%default]"), - type=int, default=True) - group.add_option("", "--output", dest="output", metavar="PATH", - help="write raw report data to PATH (or stdout if '-')", - action="store", default=None) - group.add_option("-v", "--verbose", dest="verbose", - help="show verbose test results", - action="store_true", default=False) - group.add_option("", "--exclude-stat-from-submission", dest="exclude_stat_from_submission", - help="Do not submit the stat of this type " - "[%default]", - action='append', - choices=KNOWN_SAMPLE_KEYS, - default=['hash']) - parser.add_option_group(group) - - (opts, args) = parser.parse_args(args) - if len(args) == 0: - nick = platform.uname()[1] - elif len(args) == 1: - nick, = args - else: - parser.error("invalid number of arguments") + # FIXME: an equivalent to argparse's add_argument_group is not implemented + # on click. Need to review it when such functionality is available. + # https://github.com/pallets/click/issues/373 + @staticmethod + @click.command("nt", help=usage_info, + short_help="LLVM test-suite compile and execution tests") + @click.argument("label", default=platform.uname()[1], required=False, + type=click.UNPROCESSED) + # Sandbox + @click.option("-s", "--sandbox", "sandbox_path", required=True, + help="parent directory to build and run tests in", + type=click.UNPROCESSED) + @click.option("--no-timestamp", "timestamp_build", flag_value=False, + default=True, show_default=True, + help="don't timestamp build directory (for testing)") + @click.option("--no-configure", "run_configure", flag_value=False, + default=True, show_default=True, + help="don't run configure if Makefile.config is " + "present (only useful with --no-timestamp)") + # Inputs + @click.option("--without-llvm", is_flag=True, show_default=True, + help="don't use any LLVM source or build products") + @click.option("--llvm-src", "llvm_src_root", + help="path to the LLVM source tree", + type=click.UNPROCESSED) + @click.option("--llvm-obj", "llvm_obj_root", metavar="PATH", + help="path to the LLVM source tree", + type=click.UNPROCESSED) + @click.option("--test-suite", "test_suite_root", metavar="PATH", + help="path to the LLVM test-suite sources", + type=click.UNPROCESSED) + @click.option("--test-externals", "test_suite_externals", + show_default=True, + help="path to the LLVM test-suite externals", + default='/dev/null', metavar="PATH", + type=click.UNPROCESSED) + # Test Compiler + @click.option("--cc", "cc_under_test", metavar="CC", + help="path to the C compiler to test", + type=click.UNPROCESSED) + @click.option("--cxx", "cxx_under_test", metavar="CXX", + help="path to the C++ compiler to test", + type=click.UNPROCESSED) + @click.option("--cc-reference", + help="path to the reference C compiler", + type=click.UNPROCESSED) + @click.option("--cxx-reference", + help="path to the reference C++ compiler", + type=click.UNPROCESSED) + # Test Options + @click.option("--arch", help="Set -arch in TARGET_FLAGS") + @click.option("--llvm-arch", + help="Set the ARCH value used in the makefiles to", + type=click.UNPROCESSED) + @click.option("--make-param", "make_parameters", multiple=True, + help="Add 'NAME' = 'VAL' to the makefile parameters", + type=click.UNPROCESSED) + @click.option("--isysroot", "isysroot", metavar="PATH", + help="Set -isysroot in TARGET_FLAGS", + type=click.UNPROCESSED) + @click.option("--liblto-path", metavar="PATH", + help="Specify the path to the libLTO library", + type=click.UNPROCESSED) + @click.option("--mcpu", metavar="CPU", + help="Set -mcpu in TARGET_LLCFLAGS", + type=click.UNPROCESSED) + @click.option("--relocation-model", metavar="MODEL", + help="Set -relocation-model in TARGET_LLCFLAGS", + type=click.UNPROCESSED) + @click.option("--disable-fp-elim", is_flag=True, + help="Set -disable-fp-elim in TARGET_LLCFLAGS") + @click.option("--optimize-option", show_default=True, + help="Set optimization level for {LLC_,LLI_,}OPTFLAGS", + type=click.Choice(['-O0', '-O1', '-O2', + '-O3', '-Os', '-Oz']), + default='-O3') + @click.option("--cflag", "cflags", multiple=True, + help="Additional flags to set in TARGET_FLAGS", + type=click.UNPROCESSED, default=[], metavar="FLAG") + @click.option("--cflags", "cflag_string", multiple=True, + help="Additional flags to set in TARGET_FLAGS, space " + "separated string. These flags are appended after " + "*all* the individual --cflag arguments.", + type=click.UNPROCESSED, default='', metavar="FLAG") + @click.option("--mllvm", multiple=True, + help="Add -mllvm FLAG to TARGET_FLAGS", + type=click.UNPROCESSED, default=[], metavar="FLAG") + @click.option("--spec-with-pgo", is_flag=True, help="Use PGO with SPEC") + @click.option("--build-mode", metavar="NAME", show_default=True, + default='Release+Asserts', + help="Select the LLVM build mode to use", + type=click.UNPROCESSED) + @click.option("--simple", "test_simple", is_flag=True, + help="Use TEST=simple instead of TEST=nightly") + @click.option("--test-style", type=click.Choice(['nightly', 'simple']), + default='simple', help="Set the test style to run") + @click.option("--test-time-stat", type=click.Choice(['user', 'real']), + default='user', show_default=True, + help="Set the test timing statistic to gather") + @click.option("--disable-cxx", "test_cxx", flag_value=False, default=True, + show_default=True, help="Disable C++ tests") + @click.option("--disable-externals", "test_externals", flag_value=False, + default=True, show_default=True, + help="Disable test suite externals (if configured)") + @click.option("--enable-integrated-as", "test_integrated_as", is_flag=True, + help="Enable TEST_INTEGRATED_AS tests") + @click.option("--enable-jit", "test_jit", is_flag=True, + help="Enable JIT tests") + @click.option("--disable-llc", "test_llc", + help="Disable LLC tests", + flag_value=False, show_default=True, default=True) + @click.option("--enable-llcbeta", "test_llcbeta", + help="Enable LLCBETA tests", is_flag=True) + @click.option("--disable-lto", "test_lto", + help="Disable use of link-time optimization", + flag_value=False, show_default=True, default=True) + @click.option("--small", "test_small", + help="Use smaller test inputs and disable large tests", + is_flag=True) + @click.option("--large", "test_large", + help="Use larger test inputs", is_flag=True) + @click.option("--spec-with-ref", "test_spec_ref", + help="Use reference test inputs for SPEC. " + "This is currently experimental", is_flag=True) + @click.option("--benchmarking-only", "test_benchmarking_only", + help="Benchmarking-only mode", is_flag=True) + @click.option("--only-test", "only_test", metavar="PATH", + help="Only run tests under PATH", + type=click.UNPROCESSED, default=None) + @click.option("--include-test-examples", + help="Include test module examples", + is_flag=True) + # Test Execution + @click.option("-j", "--threads", "threads", + help="Number of testing threads", + type=int, default=1, metavar="N") + @click.option("--build-threads", "build_threads", + help="Number of compilation threads", + type=int, default=0, metavar="N") + @click.option("--use-perf", type=click.UNPROCESSED, default=None, + help="Use perf to obtain high accuracy timing") + @click.option("--rerun", help="Rerun tests that have regressed.", + is_flag=True) + @click.option("--remote", is_flag=True, + help="Execute remotely, see " + "--remote-{host,port,user,client}") + @click.option("--remote-host", default="localhost", metavar="HOST", + help="Set remote execution host", type=click.UNPROCESSED) + @click.option("--remote-port", type=int, default=None, + metavar="PORT", help="Set remote execution port") + @click.option("--remote-user", help="Set remote execution user", + type=click.UNPROCESSED, default=None, metavar="USER",) + @click.option("--remote-client", type=click.UNPROCESSED, default="ssh", + help="Set remote execution client", metavar="RSH") + @click.option("--use-ios-simulator", "ios_simulator_sdk", + help=("Execute using an iOS simulator SDK (using " + "environment overrides)"), + type=click.UNPROCESSED, default=None, metavar="SDKPATH") + @click.option("--use-isolation", "use_isolation", + help=("Execute using a sandboxing profile to limit " + "OS access (e.g., to the network or " + "non-test directories)"), + is_flag=True) + @click.option("--qemu-user-mode", "qemu_user_mode", + help=("Enable qemu user mode emulation using this " + "qemu executable"), + type=click.UNPROCESSED, default=None) + @click.option("--qemu-flag", "qemu_flags", + help="Additional flags to pass to qemu", multiple=True, + type=click.UNPROCESSED, metavar="FLAG") + @click.option("--qemu-flags", "qemu_string", + help="Additional flags to pass to qemu, space " + "separated string. These flags are appended after " + "*all* the individual --qemu-flag arguments.", + type=click.UNPROCESSED, default='', metavar="FLAG") + @click.option("--multisample", "multisample", + help="Accumulate test data from multiple runs", + type=int, default=None, metavar="N") + # Output Options + @click.option("--no-auto-name", "auto_name", + help="Don't automatically derive submission name", + flag_value=False, show_default=True, default=True) + @click.option("--no-machdep-info", "use_machdep_info", + help=("Don't put machine (instance) dependent " + "variables with machine info"), + flag_value=False, show_default=True, default=True) + @click.option("--run-order", "run_order", metavar="STR", + help="String to use to identify and order this run", + type=click.UNPROCESSED, default=None) + @click.option("--machine-param", "machine_parameters", + metavar="NAME=VAL", + help="Add 'NAME' = 'VAL' to the machine parameters", + type=click.UNPROCESSED, multiple=True, default=[]) + @click.option("--run-param", "run_parameters", + metavar="NAME=VAL", + help="Add 'NAME' = 'VAL' to the run parameters", + type=click.UNPROCESSED, multiple=True, default=[]) + @click.option("--submit", "submit_url", metavar="URLORPATH", + help=("autosubmit the test result to the given server" + " (or local instance)"), + type=click.UNPROCESSED, default=[], multiple=True) + @click.option("--commit", "commit", + help="whether the autosubmit result should be committed", + type=int, default=True) + @click.option("--output", "output", metavar="PATH", + help="write raw report data to PATH (or stdout if '-')", + default=None) + @click.option("-v", "--verbose", "verbose", + help="show verbose test results", is_flag=True) + @click.option("--exclude-stat-from-submission", + "exclude_stat_from_submission", + help="Do not submit the stat of this type ", + multiple=True, type=click.Choice(KNOWN_SAMPLE_KEYS), + default=['hash']) + def cli_wrapper(*args, **kwargs): + """LLVM test-suite compile and execution tests""" + _tools_check() + nt = NTTest() + + for key, value in kwargs.items(): + setattr(nt.opts, key, value) + + results = nt.run_test(nt.opts) + nt.show_results_url(results) + + + def run_test(self, opts): + + opts.cflag_string = ' '.join(opts.cflag_string) # The --without--llvm option is the default if no LLVM paths are given. if opts.llvm_src_root is None and opts.llvm_obj_root is None: opts.without_llvm = True - # Validate options. - - if opts.sandbox_path is None: - parser.error('--sandbox is required') - - # Deprecate --simple. + # Deprecate --simple if opts.test_simple: warning("--simple is deprecated, it is the default.") del opts.test_simple @@ -1785,40 +1740,40 @@ if opts.test_style == "simple": # TEST=simple doesn't use a reference compiler. if opts.cc_reference is not None: - parser.error('--cc-reference is unused with --simple') + self._fatal('--cc-reference is unused with --simple') if opts.cxx_reference is not None: - parser.error('--cxx-reference is unused with --simple') + self._fatal('--cxx-reference is unused with --simple') # TEST=simple doesn't use a llc options. if opts.mcpu is not None: - parser.error('--mcpu is unused with --simple (use --cflag)') + self._fatal('--mcpu is unused with --simple (use --cflag)') if opts.relocation_model is not None: - parser.error('--relocation-model is unused with --simple ' - '(use --cflag)') + self._fatal('--relocation-model is unused with --simple ' + '(use --cflag)') if opts.disable_fp_elim: - parser.error('--disable-fp-elim is unused with --simple ' - '(use --cflag)') + self._fatal('--disable-fp-elim is unused with --simple ' + '(use --cflag)') else: if opts.without_llvm: - parser.error('--simple is required with --without-llvm') + self._fatal('--simple is required with --without-llvm') # Attempt to infer cc_reference and cxx_reference if not given. if opts.cc_reference is None: opts.cc_reference = which('gcc') or which('cc') if opts.cc_reference is None: - parser.error('unable to infer --cc-reference (required)') + self._fatal('unable to infer --cc-reference (required)') if opts.cxx_reference is None: opts.cxx_reference = which('g++') or which('c++') if opts.cxx_reference is None: - parser.error('unable to infer --cxx-reference (required)') + self._fatal('unable to infer --cxx-reference (required)') if opts.cc_under_test is None: - parser.error('--cc is required') + self._fatal('--cc is required') # Resolve the cc_under_test path. opts.cc_under_test = resolve_command_path(opts.cc_under_test) if not lnt.testing.util.compilers.is_valid(opts.cc_under_test): - parser.error('--cc does not point to a valid executable.') + self._fatal('--cc does not point to a valid executable.') # If there was no --cxx given, attempt to infer it from the --cc. if opts.cxx_under_test is None: @@ -1830,7 +1785,7 @@ # The cxx_under_test option is required if we are testing C++. if opts.test_cxx and opts.cxx_under_test is None: - parser.error('--cxx is required') + self._fatal('--cxx is required') if opts.cxx_under_test is not None: opts.cxx_under_test = resolve_command_path(opts.cxx_under_test) @@ -1842,11 +1797,11 @@ # Validate that the compilers under test exist. if not os.path.exists(opts.cc_under_test): - parser.error("invalid --cc argument %r, does not exist" % ( - opts.cc_under_test)) + self._fatal("invalid --cc argument %r, does not exist" % ( + opts.cc_under_test)) if not os.path.exists(opts.cxx_under_test): - parser.error("invalid --cxx argument %r, does not exist" % ( - opts.cxx_under_test)) + self._fatal("invalid --cxx argument %r, does not exist" % ( + opts.cxx_under_test)) # FIXME: As a hack to allow sampling old Clang revisions, if we are # given a C++ compiler that doesn't exist, reset it to just use the @@ -1857,55 +1812,55 @@ if opts.without_llvm: if opts.llvm_src_root is not None: - parser.error('--llvm-src is not allowed with --without-llvm') + self._fatal('--llvm-src is not allowed with --without-llvm') if opts.llvm_obj_root is not None: - parser.error('--llvm-obj is not allowed with --without-llvm') + self._fatal('--llvm-obj is not allowed with --without-llvm') else: if opts.llvm_src_root is None: - parser.error('--llvm-src is required') + self._fatal('--llvm-src is required') if opts.llvm_obj_root is None: - parser.error('--llvm-obj is required') + self._fatal('--llvm-obj is required') # Make LLVM source and object paths absolute, this is required. opts.llvm_src_root = os.path.abspath(opts.llvm_src_root) opts.llvm_obj_root = os.path.abspath(opts.llvm_obj_root) if not os.path.exists(opts.llvm_src_root): - parser.error('--llvm-src argument does not exist') + self._fatal('--llvm-src argument does not exist') if not os.path.exists(opts.llvm_obj_root): - parser.error('--llvm-obj argument does not exist') + self._fatal('--llvm-obj argument does not exist') if opts.test_suite_root is None: - parser.error('--test-suite is required') + self._fatal('--test-suite is required') elif not os.path.exists(opts.test_suite_root): - parser.error("invalid --test-suite argument, does not exist: %r" % ( - opts.test_suite_root)) + self._fatal("invalid --test-suite argument, does not exist: %r" % ( + opts.test_suite_root)) if opts.remote: if opts.remote_port is None: - parser.error('--remote-port is required with --remote') + self._fatal('--remote-port is required with --remote') if opts.remote_user is None: - parser.error('--remote-user is required with --remote') + self._fatal('--remote-user is required with --remote') else: if opts.remote_port is not None: - parser.error('--remote is required with --remote-port') + self._fatal('--remote is required with --remote-port') if opts.remote_user is not None: - parser.error('--remote is required with --remote-user') + self._fatal('--remote is required with --remote-user') if opts.spec_with_pgo and not opts.test_spec_ref: - parser.error('--spec-with-pgo is only supported with --spec-with-ref') + self._fatal('--spec-with-pgo is only supported with --spec-with-ref') # libLTO should exist, if given. if opts.liblto_path: if not os.path.exists(opts.liblto_path): - parser.error('invalid --liblto-path argument %r' % ( - opts.liblto_path,)) + self._fatal('invalid --liblto-path argument %r' % ( + opts.liblto_path,)) # Support disabling test suite externals separately from providing path. if not opts.test_externals: opts.test_suite_externals = '/dev/null' else: if not os.path.exists(opts.test_suite_externals): - parser.error( + self._fatal( "invalid --test-externals argument, does not exist: %r" % ( opts.test_suite_externals)) @@ -1936,7 +1891,7 @@ for i in range(opts.multisample): print >>sys.stderr, "%s: (multisample) running iteration %d" % ( timestamp(), i) - report = run_test(nick, i, config) + report = run_test(opts.label, i, config) reports.append(report) # Create the merged report. @@ -1958,7 +1913,7 @@ lnt_report_file.close() else: - test_results = run_test(nick, None, config) + test_results = run_test(opts.label, None, config) if opts.rerun: self.log("Performing any needed reruns.") server_report = self.submit_helper(config, commit=False) @@ -2001,8 +1956,8 @@ result = ServerUtil.submitFile(server, report_path, commit, False) except (urllib2.HTTPError, urllib2.URLError) as e: - warning("submitting to {} failed with {}".format(server, - e)) + warning("submitting to {} failed with {}".format( + server, e)) else: # Simulate a submission to retrieve the results report. # Construct a temporary database and import the result. @@ -2053,10 +2008,3 @@ status = call(["which", "tclsh"], stdout=FNULL, stderr=FNULL) if status > 0: raise SystemExit("""error: tclsh not available on your system.""") - - -def create_instance(): - _tools_check() - return NTTest() - -__all__ = ['create_instance'] Index: lnt/trunk/lnt/tests/test_suite.py =================================================================== --- lnt/trunk/lnt/tests/test_suite.py +++ lnt/trunk/lnt/tests/test_suite.py @@ -13,10 +13,9 @@ import getpass import datetime -import jinja2 from collections import defaultdict - -from optparse import OptionParser, OptionGroup +import jinja2 +import click import lnt.testing import lnt.testing.profile @@ -177,186 +176,158 @@ def describe(self): return "LLVM test-suite" - def run_test(self, name, args): - # FIXME: Add more detailed usage information - parser = OptionParser("%s [options] test-suite" % name) - - group = OptionGroup(parser, "Sandbox options") - group.add_option("-S", "--sandbox", dest="sandbox_path", - help="Parent directory to build and run tests in", - type=str, default=None, metavar="PATH") - group.add_option("", "--no-timestamp", dest="timestamp_build", - action="store_false", default=True, - help="Don't timestamp build directory (for testing)") - group.add_option("", "--no-configure", dest="run_configure", - action="store_false", default=True, - help="Don't run CMake if CMakeCache.txt is present" - " (only useful with --no-timestamp") - parser.add_option_group(group) - - group = OptionGroup(parser, "Inputs") - group.add_option("", "--test-suite", dest="test_suite_root", - type=str, metavar="PATH", default=None, - help="Path to the LLVM test-suite sources") - group.add_option("", "--test-externals", dest="test_suite_externals", - type=str, metavar="PATH", - help="Path to the LLVM test-suite externals") - group.add_option("", "--cmake-define", dest="cmake_defines", - action="append", default=[], - help=("Defines to pass to cmake. These do not require the " - "-D prefix and can be given multiple times. e.g.: " - "--cmake-define A=B => -DA=B")) - group.add_option("-C", "--cmake-cache", dest="cmake_cache", - action="append", default=[], - help=("Use one of the test-suite's cmake configurations." - " Ex: Release, Debug")) - parser.add_option_group(group) - - group = OptionGroup(parser, "Test compiler") - group.add_option("", "--cc", dest="cc", metavar="CC", - type=str, default=None, - help="Path to the C compiler to test") - group.add_option("", "--cxx", dest="cxx", metavar="CXX", - type=str, default=None, - help="Path to the C++ compiler to test (inferred from" - " --cc where possible") - group.add_option("", "--cppflags", type=str, action="append", - dest="cppflags", default=[], - help="Extra flags to pass the compiler in C or C++ mode. " - "Can be given multiple times") - group.add_option("", "--cflags", type=str, action="append", - dest="cflags", default=[], - help="Extra CFLAGS to pass to the compiler. Can be " - "given multiple times") - group.add_option("", "--cxxflags", type=str, action="append", - dest="cxxflags", default=[], - help="Extra CXXFLAGS to pass to the compiler. Can be " - "given multiple times") - parser.add_option_group(group) - - group = OptionGroup(parser, "Test selection") - group.add_option("", "--test-size", type='choice', dest="test_size", - choices=['small', 'regular', 'large'], default='regular', - help="The size of test inputs to use") - group.add_option("", "--benchmarking-only", - dest="benchmarking_only", action="store_true", - default=False, - help="Benchmarking-only mode. Disable unit tests and " - "other flaky or short-running tests") - group.add_option("", "--only-test", dest="only_test", metavar="PATH", - type=str, default=None, - help="Only run tests under PATH") - - parser.add_option_group(group) - - group = OptionGroup(parser, "Test Execution") - group.add_option("", "--only-compile", dest="only_compile", - help="Don't run the tests, just compile them.", - action="store_true", default=False, ) - group.add_option("-j", "--threads", dest="threads", - help="Number of testing (and optionally build) " - "threads", type=int, default=1, metavar="N") - group.add_option("", "--build-threads", dest="build_threads", - help="Number of compilation threads, defaults to " - "--threads", type=int, default=0, metavar="N") - group.add_option("", "--use-perf", dest="use_perf", - help=("Use Linux perf for high accuracy timing, profile " - "information or both"), - type='choice', - choices=['none', 'time', 'profile', 'all'], - default='none') - group.add_option("", "--perf-events", dest="perf_events", - help=("Define which linux perf events to measure"), - type=str, default=None) - group.add_option("", "--run-under", dest="run_under", - help="Wrapper to run tests under ['%default']", - type=str, default="") - group.add_option("", "--exec-multisample", dest="exec_multisample", - help="Accumulate execution test data from multiple runs", - type=int, default=1, metavar="N") - group.add_option("", "--compile-multisample", dest="compile_multisample", - help="Accumulate compile test data from multiple runs", - type=int, default=1, metavar="N") - group.add_option("-d", "--diagnose", dest="diagnose", - help="Produce a diagnostic report for a particular " - "test, this will not run all the tests. Must be" - " used in conjunction with --only-test.", - action="store_true", default=False,) - group.add_option("", "--pgo", dest="pgo", - help="Run the test-suite in training mode first and" - " collect PGO data, then rerun with that training " - "data.", - action="store_true", default=False,) - - parser.add_option_group(group) - - group = OptionGroup(parser, "Output Options") - group.add_option("", "--no-auto-name", dest="auto_name", - help="Don't automatically derive submission name", - action="store_false", default=True) - group.add_option("", "--run-order", dest="run_order", metavar="STR", - help="String to use to identify and order this run", - action="store", type=str, default=None) - group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH", - help=("autosubmit the test result to the given server" - " (or local instance) [%default]"), - type=str, default=None) - group.add_option("", "--commit", dest="commit", - help=("whether the autosubmit result should be committed " - "[%default]"), - type=int, default=True) - group.add_option("", "--succinct-compile-output", - help="run Make without VERBOSE=1", - action="store_true", dest="succinct") - group.add_option("-v", "--verbose", dest="verbose", - help="show verbose test results", - action="store_true", default=False) - group.add_option("", "--exclude-stat-from-submission", - dest="exclude_stat_from_submission", - help="Do not submit the stat of this type [%default]", - action='append', choices=KNOWN_SAMPLE_KEYS, - type='choice', default=[]) - group.add_option("", "--single-result", dest="single_result", - help=("only execute this single test and apply " - "--single-result-predicate to calculate the " - "exit status")) - group.add_option("", "--single-result-predicate", - dest="single_result_predicate", - help=("the predicate to apply to calculate the exit " - "status (with --single-result)"), - default="status") - parser.add_option_group(group) - - group = OptionGroup(parser, "Test tools") - group.add_option("", "--use-cmake", dest="cmake", metavar="PATH", - type=str, default="cmake", - help="Path to CMake [cmake]") - group.add_option("", "--use-make", dest="make", metavar="PATH", - type=str, default="make", - help="Path to Make [make]") - group.add_option("", "--use-lit", dest="lit", metavar="PATH", - type=str, default="llvm-lit", - help="Path to the LIT test runner [llvm-lit]") - parser.add_option_group(group) - - (opts, args) = parser.parse_args(args) - self.opts = opts - - if len(args) == 0: - self.nick = platform.uname()[1] - elif len(args) == 1: - self.nick = args[0] - else: - parser.error("Expected no positional arguments (got: %r)" % (args,)) + @staticmethod + @click.command("test-suite") + @click.argument("label", default=platform.uname()[1], required=False, + type=click.UNPROCESSED) + # Sandbox options + @click.option("-S", "--sandbox", "sandbox_path", required=True, + help="Parent directory to build and run tests in", + type=click.UNPROCESSED, metavar="PATH") + @click.option("--no-timestamp", "timestamp_build", + flag_value=False, default=True, + help="Don't timestamp build directory (for testing)") + @click.option("--no-configure", "run_configure", + flag_value=False, default=True, + help="Don't run CMake if CMakeCache.txt is present" + " (only useful with --no-timestamp") + # Inputs + @click.option("--test-suite", "test_suite_root", + type=click.UNPROCESSED, metavar="PATH", + help="Path to the LLVM test-suite sources") + @click.option("--test-externals", "test_suite_externals", + type=click.UNPROCESSED, metavar="PATH", + help="Path to the LLVM test-suite externals") + @click.option("--cmake-define", "cmake_defines", + multiple=True, + help="Defines to pass to cmake. These do not require the " + "-D prefix and can be given multiple times. e.g.: " + "--cmake-define A=B => -DA=B") + @click.option("-C", "--cmake-cache", "cmake_cache", multiple=True, + default=[], + help="Use one of the test-suite's cmake configurations." + " Ex: Release, Debug") + # Test compiler + @click.option("--cc", "cc", metavar="CC", type=click.UNPROCESSED, + default=None, + help="Path to the C compiler to test") + @click.option("--cxx", "cxx", metavar="CXX", type=click.UNPROCESSED, + default=None, + help="Path to the C++ compiler to test (inferred from" + " --cc where possible") + @click.option("--cppflags", "cppflags", type=click.UNPROCESSED, + multiple=True, default=[], + help="Extra flags to pass the compiler in C or C++ mode. " + "Can be given multiple times") + @click.option("--cflags", "--cflag", "cflags", type=click.UNPROCESSED, + multiple=True, default=[], + help="Extra CFLAGS to pass to the compiler. Can be " + "given multiple times") + @click.option("--cxxflags", "cxxflags", type=click.UNPROCESSED, + multiple=True, default=[], + help="Extra CXXFLAGS to pass to the compiler. Can be " + "given multiple times") + # Test selection + @click.option("--test-size", "test_size", + type=click.Choice(['small', 'regular', 'large']), + default='regular', help="The size of test inputs to use") + @click.option("--benchmarking-only", "benchmarking_only", is_flag=True, + help="Benchmarking-only mode. Disable unit tests and " + "other flaky or short-running tests") + @click.option("--only-test", "only_test", metavar="PATH", + type=click.UNPROCESSED, default=None, + help="Only run tests under PATH") + # Test Execution + @click.option("--only-compile", "only_compile", + help="Don't run the tests, just compile them.", is_flag=True) + @click.option("-j", "--threads", "threads", + help="Number of testing (and optionally build) " + "threads", type=int, default=1, metavar="N") + @click.option("--build-threads", "build_threads", + help="Number of compilation threads, defaults to --threads", + type=int, default=0, metavar="N") + @click.option("--use-perf", "use_perf", + help="Use Linux perf for high accuracy timing, profile " + "information or both", + type=click.Choice(['none', 'time', 'profile', 'all']), + default='none') + @click.option("--perf-events", "perf_events", + help=("Define which linux perf events to measure"), + type=click.UNPROCESSED, default=None) + @click.option("--run-under", "run_under", default="", + help="Wrapper to run tests under", type=click.UNPROCESSED) + @click.option("--exec-multisample", "exec_multisample", + help="Accumulate execution test data from multiple runs", + type=int, default=1, metavar="N") + @click.option("--compile-multisample", "compile_multisample", + help="Accumulate compile test data from multiple runs", + type=int, default=1, metavar="N") + @click.option("-d", "--diagnose", "diagnose", + help="Produce a diagnostic report for a particular " + "test, this will not run all the tests. Must be" + " used in conjunction with --only-test.", + is_flag=True, default=False,) + @click.option("--pgo", "pgo", + help="Run the test-suite in training mode first and" + " collect PGO data, then rerun with that training " + "data.", + is_flag=True, default=False,) + # Output Options + @click.option("--no-auto-name", "auto_name", + help="Don't automatically derive submission name", + flag_value=False, default=True) + @click.option("--run-order", "run_order", metavar="STR", + help="String to use to identify and order this run") + @click.option("--submit", "submit_url", metavar="URLORPATH", + help="autosubmit the test result to the given server" + " (or local instance)", + type=click.UNPROCESSED, default=None) + @click.option("--commit", "commit", + help="whether the autosubmit result should be committed", + type=int, default=True) + @click.option("--succinct-compile-output", "succinct", + help="run Make without VERBOSE=1", is_flag=True) + @click.option("-v", "--verbose", "verbose", is_flag=True, default=False, + help="show verbose test results") + @click.option("--exclude-stat-from-submission", + "exclude_stat_from_submission", + help="Do not submit the stat of this type", + multiple=True, default=[], + type=click.Choice(KNOWN_SAMPLE_KEYS)) + @click.option("--single-result", "single_result", + help="only execute this single test and apply " + "--single-result-predicate to calculate the exit " + "status") + @click.option("--single-result-predicate", "single_result_predicate", + help="the predicate to apply to calculate the exit " + "status (with --single-result)", default="status") + # Test tools + @click.option("--use-cmake", "cmake", metavar="PATH", + type=click.UNPROCESSED, default="cmake", + help="Path to CMake [cmake]") + @click.option("--use-make", "make", metavar="PATH", + type=click.UNPROCESSED, default="make", + help="Path to Make [make]") + @click.option("--use-lit", "lit", metavar="PATH", type=click.UNPROCESSED, + default="llvm-lit", + help="Path to the LIT test runner [llvm-lit]") + def cli_wrapper(*args, **kwargs): + """LLVM test-suite""" + test_suite = TestSuiteTest() + + for key, value in kwargs.items(): + setattr(test_suite.opts, key, value) - if self.opts.sandbox_path is None: - parser.error('--sandbox is required') + results = test_suite.run_test(test_suite.opts) + test_suite.show_results_url(results) + + def run_test(self, opts): if self.opts.cc is not None: self.opts.cc = resolve_command_path(self.opts.cc) if not lnt.testing.util.compilers.is_valid(self.opts.cc): - parser.error('--cc does not point to a valid executable.') + self._fatal('--cc does not point to a valid executable.') # If there was no --cxx given, attempt to infer it from the --cc. if self.opts.cxx is None: @@ -366,41 +337,41 @@ note("Inferred C++ compiler under test as: %r" % (self.opts.cxx,)) else: - parser.error("unable to infer --cxx - set it manually.") + self._fatal("unable to infer --cxx - set it manually.") else: self.opts.cxx = resolve_command_path(self.opts.cxx) if not os.path.exists(self.opts.cxx): - parser.error("invalid --cxx argument %r, does not exist" - % (self.opts.cxx)) + self._fatal("invalid --cxx argument %r, does not exist" + % (self.opts.cxx)) if opts.test_suite_root is None: - parser.error('--test-suite is required') + self._fatal('--test-suite is required') if not os.path.exists(opts.test_suite_root): - parser.error("invalid --test-suite argument, does not exist: %r" % ( + self._fatal("invalid --test-suite argument, does not exist: %r" % ( opts.test_suite_root)) if opts.test_suite_externals: if not os.path.exists(opts.test_suite_externals): - parser.error( + self._fatal( "invalid --test-externals argument, does not exist: %r" % ( opts.test_suite_externals,)) opts.cmake = resolve_command_path(opts.cmake) if not isexecfile(opts.cmake): - parser.error("CMake tool not found (looked for %s)" % opts.cmake) + self._fatal("CMake tool not found (looked for %s)" % opts.cmake) opts.make = resolve_command_path(opts.make) if not isexecfile(opts.make): - parser.error("Make tool not found (looked for %s)" % opts.make) + self._fatal("Make tool not found (looked for %s)" % opts.make) opts.lit = resolve_command_path(opts.lit) if not isexecfile(opts.lit): - parser.error("LIT tool not found (looked for %s)" % opts.lit) + self._fatal("LIT tool not found (looked for %s)" % opts.lit) if opts.run_under: split = shlex.split(opts.run_under) split[0] = resolve_command_path(split[0]) if not isexecfile(split[0]): - parser.error("Run under wrapper not found (looked for %s)" % - opts.run_under) + self._fatal("Run under wrapper not found (looked for %s)" % + opts.run_under) if opts.single_result: # --single-result implies --only-test @@ -419,12 +390,12 @@ opts.only_test = (os.path.dirname(opts.only_test), os.path.basename(opts.only_test)) else: - parser.error("--only-test argument not understood (must be a " + - " test or directory name)") + self._fatal("--only-test argument not understood (must be a " + + " test or directory name)") if opts.single_result and not opts.only_test[1]: - parser.error("--single-result must be given a single test name, not a " + - "directory name") + self._fatal("--single-result must be given a single test name, not a " + + "directory name") opts.cppflags = ' '.join(opts.cppflags) opts.cflags = ' '.join(opts.cflags) @@ -432,7 +403,7 @@ if opts.diagnose: if not opts.only_test: - parser.error("--diagnose requires --only-test") + self._fatal("--diagnose requires --only-test") self.start_time = timestamp() @@ -461,7 +432,7 @@ cmake_vars = self._extract_cmake_vars_from_cache() if "CMAKE_C_COMPILER" not in cmake_vars or \ not os.path.exists(cmake_vars["CMAKE_C_COMPILER"]): - parser.error( + self._fatal( "Couldn't find C compiler (%s). Maybe you should specify --cc?" % cmake_vars.get("CMAKE_C_COMPILER")) @@ -476,9 +447,8 @@ # Construct the nickname from a few key parameters. cc_info = self._get_cc_info(cmake_vars) cc_nick = '%s_%s' % (cc_info['cc_name'], cc_info['cc_build']) - self.nick += "__%s__%s" % (cc_nick, - cc_info['cc_target'].split('-')[0]) - note('Using nickname: %r' % self.nick) + opts.label += "__%s__%s" % (cc_nick, cc_info['cc_target'].split('-')[0]) + note('Using nickname: %r' % opts.label) # When we can't detect the clang version we use 0 instead. That # is a horrible failure mode because all of our data ends up going @@ -636,7 +606,7 @@ if 'TEST_SUITE_RUN_TYPE' not in defs: defs['TEST_SUITE_RUN_TYPE'] = 'ref' - for item in self.opts.cmake_defines + extra_cmake_defs: + for item in tuple(self.opts.cmake_defines) + tuple(extra_cmake_defs): k, v = item.split('=', 1) # make sure the overriding of the settings above also works # when the cmake-define-defined variable has a datatype @@ -769,7 +739,7 @@ 'XFAIL': lnt.testing.XFAIL, 'XPASS': lnt.testing.FAIL, 'UNRESOLVED': lnt.testing.FAIL - }[code] + }[code] def _test_failed_to_compile(self, raw_name, path): # FIXME: Do we need to add ".exe" in windows? @@ -882,7 +852,7 @@ if k == 'link_time': # Move link time into a second benchmark's compile-time. - server_name = name + '-link.' + LIT_METRIC_TO_LNT[k] + server_name = name + '-link.' + LIT_METRIC_TO_LNT[k] test_samples.append( lnt.testing.TestSamples(server_name, @@ -894,8 +864,7 @@ test_samples.append( lnt.testing.TestSamples(name + '.compile.status', [lnt.testing.FAIL], - test_info), - ) + test_info)) elif not is_pass: test_samples.append( @@ -937,7 +906,7 @@ machine_info = { } - machine = lnt.testing.Machine(self.nick, machine_info) + machine = lnt.testing.Machine(self.opts.label, machine_info) run = lnt.testing.Run(self.start_time, timestamp(), info=run_info) report = lnt.testing.Report(machine, run, test_samples) return report @@ -1100,8 +1069,7 @@ warning("Tests may fail because of iprofiler's output.") # The dtps file will be saved as root, make it so # that we can read it. - chmod = sudo + ["chown", "-R", getpass.getuser(), - short_name + ".dtps"] + chmod = sudo + ["chown", "-R", getpass.getuser(), short_name + ".dtps"] subprocess.call(chmod) profile = local_path + "/" + short_name + ".dtps" shutil.copytree(profile, report_path + "/" + short_name + ".dtps") @@ -1121,9 +1089,3 @@ return report_path return DontSubmitResults() - - -def create_instance(): - return TestSuiteTest() - -__all__ = ['create_instance'] Index: lnt/trunk/lnt/util/multitool.py =================================================================== --- lnt/trunk/lnt/util/multitool.py +++ lnt/trunk/lnt/util/multitool.py @@ -1,78 +0,0 @@ -import os -import sys - -class MultiTool(object): - """ - This object defines a generic command line tool instance, which dynamically - builds its commands from a module dictionary. - - Example usage:: - - import multitool - - def action_foo(name, args): - "the foo command" - - ... - - tool = multitool.MultiTool(locals()) - if __name__ == '__main__': - tool.main(sys.argv) - - Any function beginning with "action_" is considered a tool command. It's - name is defined by the function name suffix. Underscores in the function - name are converted to '-' in the command line syntax. Actions ending ith - "-debug" are not listed in the help. - """ - - def __init__(self, locals, version=None): - self.version = version - - # Create the list of commands. - self.commands = dict((name[7:].replace('_','-'), f) - for name,f in locals.items() - if name.startswith('action_')) - - def usage(self, name): - print >>sys.stderr, "Usage: %s [options] ... arguments ..." %( - os.path.basename(name),) - print >>sys.stderr - print >>sys.stderr, """\ -Use ``%s --help`` for more information on a specific command.\n""" % ( - os.path.basename(name),) - print >>sys.stderr, "Available commands:" - cmds_width = max(map(len, self.commands)) - for name,func in sorted(self.commands.items()): - if name.endswith("-debug"): - continue - - print >>sys.stderr, " %-*s - %s" % (cmds_width, name, func.__doc__) - sys.exit(1) - - def main(self, args=None): - if args is None: - args = sys.argv - - progname = os.path.basename(args.pop(0)) - - # Parse immediate command line options. - while args and args[0].startswith("-"): - option = args.pop(0) - if option in ("-h", "--help"): - self.usage(progname) - elif option in ("-v", "--version") and self.version is not None: - print self.version - return - else: - print >>sys.stderr, "error: invalid option %r\n" % (option,) - self.usage(progname) - - if not args: - self.usage(progname) - - cmd = args.pop(0) - if cmd not in self.commands: - print >>sys.stderr,"error: invalid command %r\n" % cmd - self.usage(progname) - - self.commands[cmd]('%s %s' % (progname, cmd), args) Index: lnt/trunk/requirements.client.txt =================================================================== --- lnt/trunk/requirements.client.txt +++ lnt/trunk/requirements.client.txt @@ -15,4 +15,5 @@ wsgiref==0.1.2 WTForms==2.0.2 Flask-WTF==0.12 -typing \ No newline at end of file +typing +click==6.7