[LNT] Added the ability to disable regressions processing to improve performance

[kpdev42]

Summary

The regresssion processing can be disabled on the server side or per submit.
Supported the option ignore_regressions = True in lnt.cfg to disable regresssion processing at all.
Added the parameter --ignore-regressions to lnt submit command.

[thopre]

LGTM. Gee that's an impressive amount of changes needed for bubbling up the config and command-line options to the processing code!

[cmatthews]

Great idea! They can be very slow on large DBs to process.

[dmgreen]

lnt/lnttool/main.py

191

Should this still include testsuite?

[kpdev42]

lnt/lnttool/main.py

191

Thank you! Fixed https://reviews.llvm.org/rLNTbe3643cefc52cb74373ae1b0bc5c62391f4f73b6

[kpdev42]

lnt/lnttool/main.py

191

I meant that missing testsuite was fixed

lnt/lnttool/main.py

 @click.command("submit")
 @click.argument("url")
 @click.argument("files", nargs=-1, type=click.Path(exists=True), required=True)
 @submit_options
 @click.option("--verbose", "-v", is_flag=True,
               help="show verbose test results")
 @click.option("--testsuite", "-s", default='nts', show_default=True,
               help="testsuite to use in case the url is a file path")
-def action_submit(url, files, select_machine, merge, verbose, testsuite):
[e-mail]("--ignore-regressions", is_flag=True,
+              help="disable regression tracking")
+def action_submit(url, files, select_machine, merge, verbose,
+                  ignore_regressions):
     """submit a test report to the server"""
     from lnt.util import ServerUtil
     import lnt.util.ImportData
 
     if '://' not in url:
         init_logger(logging.DEBUG)
         register_hooks()
 
     results = ServerUtil.submitFiles(url, files, verbose,
                                      select_machine=select_machine,
-                                     merge_run=merge, testsuite=testsuite)
+                                     merge_run=merge, testsuite=testsuite,
+                                     ignore_regressions=ignore_regressions)
     for submitted_file in results:
         if verbose:
             lnt.util.ImportData.print_report_result(
                 submitted_file, sys.stdout, sys.stderr, True)
         _print_result_url(submitted_file, verbose)
     if len(files) != len(results):
         sys.exit(1)
 

lnt/server/config.py

def from_data(path, data):
         blacklist = data.get('blacklist', None)
         api_auth_token = data.get('api_auth_token', None)
         if blacklist and baseDir:
             blacklist = os.path.join(baseDir, blacklist)
         else:
             blacklist = None
         secretKey = data.get('secret_key', None)
 
+        ignore_regressions = data.get('ignore_regressions', False)
+
         return Config(data.get('name', 'LNT'), data['zorgURL'],
                       dbDir, os.path.join(baseDir, tempDir),
                       os.path.join(baseDir, profileDir), secretKey,
                       dict([(k, DBInfo.from_data(dbDirPath, v,
                                                  default_email_config,
                                                  0))
                            for k, v in data['databases'].items()]),
-                      blacklist, schemasDir, api_auth_token)
+                      blacklist, schemasDir, api_auth_token, ignore_regressions)
 
     @staticmethod
     def dummy_instance():
         baseDir = tempfile.mkdtemp()
         dbDir = '.'
         profileDirPath = os.path.join(baseDir, 'profiles')
         tempDir = os.path.join(baseDir, 'tmp')
         schemasDir = os.path.join(baseDir, 'schemas')
def __init__(self,
                  zorgURL,
                  dbDir,
                  tempDir,
                  profileDir,
                  secretKey,
                  databases,
                  blacklist,
                  schemasDir,
-                 api_auth_token=None):
+                 api_auth_token=None,
+                 ignore_regressions=False):
         self.name = name
         self.zorgURL = zorgURL
         self.dbDir = dbDir
         self.tempDir = tempDir
         self.secretKey = secretKey
         self.blacklist = blacklist
         self.profileDir = profileDir
         self.schemasDir = schemasDir
         while self.zorgURL.endswith('/'):
             self.zorgURL = zorgURL[:-1]
         self.databases = databases
         for db in self.databases.values():
             db.config = self
         self.api_auth_token = api_auth_token
+        self.ignore_regressions = ignore_regressions
 
     def get_database(self, name):
         """
         get_database(name) -> db or None
 
         Return the appropriate instance of the database with the given name, or
         None if there is no database with that name."""
 

lnt/server/ui/api.py

class Runs(Resource):
     @requires_auth_token
     def post():
         """Add a new run into the lnt database"""
         session = request.session
         db = request.get_db()
         data = request.get_data(as_text=True)
         select_machine = request.values.get('select_machine', 'match')
         merge = request.values.get('merge', None)
+        ignore_regressions = request.values.get('ignore_regressions', False) \
+            or getattr(current_app.old_config, 'ignore_regressions', False)
         result = lnt.util.ImportData.import_from_string(
             current_app.old_config, g.db_name, db, session, g.testsuite_name,
-            data, select_machine=select_machine, merge_run=merge)
+            data, select_machine=select_machine, merge_run=merge,
+            ignore_regressions=ignore_regressions)
 
         error = result['error']
         if error is not None:
             response = jsonify(result)
             response.status = '400'
             logger.warning("%s: Submission rejected: %s" %
                            (request.url, error))
             return response

lnt/server/ui/views.py

def _do_submit():
     if 'select_machine' not in request.form and \
             'update_machine' in request.form:
         # Compatibility with old clients
         update_machine = int(request.form.get('update_machine', 0)) != 0
         select_machine = 'update' if update_machine else 'match'
     else:
         select_machine = request.form.get('select_machine', 'match')
     merge_run = request.form.get('merge', None)
+    ignore_regressions = request.form.get('ignore_regressions', False) \
+        or getattr(current_app.old_config, 'ignore_regressions', False)
 
     if input_file and not input_file.content_length:
         input_file = None
 
     if not input_file and not input_data:
         return render_template(
             "submit_run.html", error="must provide input file or data")
     if input_file and input_data:
if g.testsuite_name is None:
         g.testsuite_name = 'nts'
 
     # Get a DB connection.
     session = request.session
     db = request.get_db()
 
     result = lnt.util.ImportData.import_from_string(
         current_app.old_config, g.db_name, db, session, g.testsuite_name,
-        data_value, select_machine=select_machine, merge_run=merge_run)
+        data_value, select_machine=select_machine, merge_run=merge_run,
+        ignore_regressions=ignore_regressions)
 
     # It is nice to have a full URL to the run, so fixup the request URL
     # here were we know more about the flask instance.
     if result.get('result_url'):
         result['result_url'] = request.url_root + result['result_url']
 
     response = flask.jsonify(**result)
     error = result['error']

lnt/util/ImportData.py

 import time
 
 from lnt.server.db import fieldchange
 
 
 def import_and_report(config, db_name, db, session, file, format, ts_name,
                       show_sample_count=False, disable_email=False,
                       disable_report=False, select_machine=None,
-                      merge_run=None):
+                      merge_run=None, ignore_regressions=False):
     """
     import_and_report(config, db_name, db, session, file, format, ts_name,
                       [show_sample_count], [disable_email],
                       [disable_report], [select_machine], [merge_run])
                      -> ... object ...
 
     Import a test data file into an LNT server and generate a test report. On
     success, run is the newly imported run.
def import_and_report(config, db_name, db, session, file, format, ts_name,
     result['added_tests'] = ts.getNumTests(session) - numTests
     if show_sample_count:
         result['added_samples'] = ts.getNumSamples(session) - numSamples
 
     result['committed'] = True
     result['run_id'] = run.id
     session.commit()
 
+    if ignore_regressions:
+        logger.info("Regenerating regressions skipped")
+    else:
         fieldchange.post_submit_tasks(session, ts, run.id)
 
     # Add a handy relative link to the submitted run.
     result['result_url'] = "db_{}/v4/{}/{}".format(db_name, ts_name, run.id)
     result['report_time'] = time.time() - importStartTime
     result['total_time'] = time.time() - startTime
     logger.info("Successfully created {}".format(result['result_url']))
     # If this database has a shadow import configured, import the run into that
     # database as well.
if config and config.databases[db_name].shadow_import:
             # Perform the shadow import.
             shadow_session = shadow_db.make_session()
             shadow_result = import_and_report(config, shadow_name,
                                               shadow_db, shadow_session, file,
                                               format, ts_name,
                                               show_sample_count, disable_email,
                                               disable_report,
                                               select_machine=select_machine,
-                                              merge_run=merge_run)
+                                              merge_run=merge_run,
+                                              ignore_regressions=ignore_regressions)
 
             # Append the shadow result to the result.
             result['shadow_result'] = shadow_result
 
     result['success'] = True
     return result
 
 
if total_added:
         print(file=out)
     print("Results", file=out)
     print("----------------", file=out)
     for kind, count in result_kinds.items():
         print(kind, ":", count, file=out)
 
 
 def import_from_string(config, db_name, db, session, ts_name, data,
-                       select_machine=None, merge_run=None):
+                       select_machine=None, merge_run=None,
+                       ignore_regressions=False):
     # Stash a copy of the raw submission.
     #
     # To keep the temporary directory organized, we keep files in
     # subdirectories organized by (database, year-month).
     utcnow = datetime.datetime.utcnow()
     tmpdir = os.path.join(config.tempDir, db_name,
                           "%04d-%02d" % (utcnow.year, utcnow.month))
     try:
def import_from_string(config, db_name, db, session, ts_name, data,
 
     # Import the data.
     #
     # FIXME: Gracefully handle formats failures and DOS attempts. We
     # should at least reject overly large inputs.
 
     result = lnt.util.ImportData.import_and_report(
         config, db_name, db, session, path, '<auto>', ts_name,
-        select_machine=select_machine, merge_run=merge_run)
+        select_machine=select_machine, merge_run=merge_run,
+        ignore_regressions=ignore_regressions)
     return result

lnt/util/ServerUtil.py

except ValueError:
         print("error: {}".format(reply))
         return
     sys.stderr.write("error: lnt server: {}\n".format(error.get('error')))
     message = error.get('message', '')
     if message:
         sys.stderr.write(message + '\n')
 
 
-def submitFileToServer(url, file, select_machine=None, merge_run=None):
+def submitFileToServer(url, file, select_machine=None, merge_run=None,
+                       ignore_regressions=False):
     with open(file, 'rb') as f:
         values = {
             'input_data': f.read(),
             'commit': "1",  # compatibility with old servers.
         }
         if select_machine is not None:
             values['select_machine'] = select_machine
         if merge_run is not None:
             values['merge'] = merge_run
+        if ignore_regressions:
+            values['ignore_regressions'] = True
     headers = {'Accept': 'application/json'}
     data = urllib.parse.urlencode(values).encode(encoding='ascii')
     try:
         context = ssl.create_default_context(cafile=certifi.where())
         response = urllib.request.urlopen(urllib.request.Request(url, data, headers=headers), context=context)
     except urllib.error.HTTPError as e:
         _show_json_error(e.read())
         return
except Exception:
         traceback.print_exc()
         print()
         print("Result:")
         print("error:", result_data)
         return
 
 
 def submitFileToInstance(path, file, select_machine=None, merge_run=None,
-                         testsuite=None):
+                         testsuite=None, ignore_regressions=False):
     # Otherwise, assume it is a local url and submit to the default database
     # in the instance.
     instance = lnt.server.instance.Instance.frompath(path)
     config = instance.config
     db_name = 'default'
     with contextlib.closing(config.get_database(db_name)) as db:
         if db is None:
             raise ValueError("no default database in instance: %r" % (path,))
         session = db.make_session()
         return lnt.util.ImportData.import_and_report(
             config, db_name, db, session, file, format='<auto>',
             ts_name=testsuite or 'nts', select_machine=select_machine,
-            merge_run=merge_run)
+            merge_run=merge_run, ignore_regressions=ignore_regressions)
 
 
 def submitFile(url, file, verbose, select_machine=None, merge_run=None,
-               testsuite=None):
+               testsuite=None, ignore_regressions=False):
     # If this is a real url, submit it using urllib.
     if '://' in url:
-        result = submitFileToServer(url, file, select_machine, merge_run)
+        result = submitFileToServer(url, file, select_machine, merge_run,
+                                    ignore_regressions)
     else:
         result = submitFileToInstance(url, file, select_machine, merge_run,
-                                      testsuite)
+                                      testsuite, ignore_regressions)
     return result
 
 
 def submitFiles(url, files, verbose, select_machine=None, merge_run=None,
-                testsuite=None):
+                testsuite=None, ignore_regressions=False):
     results = []
     for file in files:
         result = submitFile(url, file, verbose, select_machine=select_machine,
-                            merge_run=merge_run, testsuite=testsuite)
+                            merge_run=merge_run, testsuite=testsuite,
+                            ignore_regressions=ignore_regressions)
         if result:
             results.append(result)
     return results