test_suite: Add a --just-submit flag and mode

[MatzeB]

Summary

Add a --just-submit=xxx.json option to "lnt runtest test-suite". When this option is used the cmake/make/lit steps are skipped and just the submitted xxx.json lit result is submitted to the lnt server.

[MatzeB]

I should also mention that this commit rewrites some sketchy detection of test compilation failures which would previously look for .test files - unfortuntately those exist even if the compilation failed. Instead I look for the newly introduced 'NOEXE' flag from the lit output now. (I did this as part of this patch because I wanted to get rid of any filesystem activity outside of reading the xxx.json file in --just-submit mode).

[cmatthews]

I agree with the updated test parsing logic, that is much better.

This flag seems to be redundant to the "lnt submit" feature which is supposed to do exactly the same thing. I would guess that does not work with the lit json directly though? It seems to me that would be a better place for this functionality, otherwise we have lnt runtest test-suite --just-submit <json> and lnt submit <json> which do the same thing to different kinds of json files.

[MatzeB]

In D19949#421849, @cmatthews wrote:

I agree with the updated test parsing logic, that is much better.

This flag seems to be redundant to the "lnt submit" feature which is supposed to do exactly the same thing. I would guess that does not work with the lit json directly though? It seems to me that would be a better place for this functionality, otherwise we have lnt runtest test-suite --just-submit <json> and lnt submit <json> which do the same thing to different kinds of json files.

Yes that sounds like the better approach in terms of usability. I just have to find a way to refactor this because it currently relies on some code from test-suite.py/builtintest.py to fill in some information about the compiler and run orders which isn't available in the lit output.

[MatzeB]

Abandoning this in favor of D34132

lnt/tests/test_suite.py

def run_test(self, name, args):
                          action="store_false", default=True)
         group.add_option("", "--run-order", dest="run_order", metavar="STR",
                          help="String to use to identify and order this run",
                          action="store", type=str, default=None)
         group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH",
                          help=("autosubmit the test result to the given server"
                                " (or local instance) [%default]"),
                          type=str, default=None)
+        group.add_option("", "--just-submit", dest="just_submit",
+                         metavar="LITOUTPUT",
+                         help="Just submit the specified lit result file. Do " +
+                              " not run cmake and lit", type=str, default=None)
         group.add_option("", "--commit", dest="commit",
                          help=("whether the autosubmit result should be committed "
                                "[%default]"),
                          type=int, default=True)
         group.add_option("", "--succinct-compile-output",
                          help="run Make without VERBOSE=1",
                          action="store_true", dest="succinct")
         group.add_option("-v", "--verbose", dest="verbose",
def run_test(self, name, args):
 
         if len(args) == 0:
             self.nick = platform.uname()[1]
         elif len(args) == 1:
             self.nick = args[0]
         else:
             parser.error("Expected no positional arguments (got: %r)" % (args,))
 
+        if self.opts.just_submit is not None:
+            # Just parse the given lit result file.
+            self.start_time = timestamp()
+            data = json.load(open(self.opts.just_submit))
+            report = self._parse_lit_output(data)
+            report_file = tempfile.NamedTemporaryFile(suffix='.json',
+                                                      delete=False)
+            report_file.write(report.render())
+            report_file.close()
+            return self.submit(report_file.name, self.opts, commit=True)
+
         for a in ['cross_compiling', 'cross_compiling_system_name', 'llvm_arch']:
             if getattr(opts, a):
                 parser.error('option "%s" is not yet implemented!' % a)
             
         if self.opts.sandbox_path is None:
             parser.error('--sandbox is required')
 
         if self.opts.cc is None:
class TestSuiteTest(BuiltinTest):
     def _get_lnt_code(self, code):
         return {'PASS': lnt.testing.PASS,
                 'FAIL': lnt.testing.FAIL,
                 'XFAIL': lnt.testing.XFAIL,
                 'XPASS': lnt.testing.FAIL,
                 'UNRESOLVED': lnt.testing.FAIL
                 }[code]
     
-    def _test_failed_to_compile(self, raw_name, path):
-        # FIXME: Do we need to add ".exe" in windows?
-        name = raw_name.rsplit('.test', 1)[0]
-        return not os.path.exists(os.path.join(path, name))
-
     def _get_target_flags(self):
         return shlex.split(self.opts.cppflags + self.opts.cflags)
     
     def _get_cc_info(self):
         return lnt.testing.util.compilers.get_cc_info(self.opts.cc,
                                                       self._get_target_flags())
 
-    def _parse_lit_output(self, path, data, only_test=False):
+    def _parse_lit_output(self, data, only_test=False):
         LIT_METRIC_TO_LNT = {
             'compile_time': 'compile',
             'exec_time': 'exec',
             'score': 'score',
             'hash': 'hash'
         }
         LIT_METRIC_CONV_FN = {
             'compile_time': float,
def _parse_lit_output(self, data, only_test=False):
         if only_test:
             ignore.append('compile')
 
         profiles_to_import = []
             
         for test_data in data['tests']:
             raw_name = test_data['name'].split(' :: ', 1)[1]
             name = 'nts.' + raw_name.rsplit('.test', 1)[0]
-            is_pass = self._is_pass_code(test_data['code'])
+            code = test_data['code']
+            is_pass = self._is_pass_code(code)
 
             # If --single-result is given, exit based on --single-result-predicate
             if self.opts.single_result and \
                raw_name == self.opts.single_result + '.test':
                 env = {'status': is_pass}
                 if 'metrics' in test_data:
                     for k, v in test_data['metrics'].items():
                         env[k] = v
def _parse_lit_output(self, data, only_test=False):
                     if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[k] in ignore:
                         continue
                     test_samples.append(
                         lnt.testing.TestSamples(name + '.' + LIT_METRIC_TO_LNT[k],
                                                 [v],
                                                 test_info,
                                                 LIT_METRIC_CONV_FN[k]))
 
-            if self._test_failed_to_compile(raw_name, path):
-                test_samples.append(
-                    lnt.testing.TestSamples(name + '.compile.status',
-                                            [lnt.testing.FAIL],
-                                            test_info))
+            if code != 'NOEXE':
+                compile_status = lnt.testing.PASS
+            else:
+                compile_status = lnt.testing.FAIL
+                                        [compile_status], test_info))
 
-            elif not is_pass:
-                test_samples.append(
-                    lnt.testing.TestSamples(name + '.exec.status',
-                                            [self._get_lnt_code(test_data['code'])],
-                                            test_info))
+            if is_pass:
+                exec_status = lnt.testing.PASS
+            else:
+                exec_status = lnt.testing.FAIL
+                                        [self._get_lnt_code(code)],
 
         # Now import the profiles in parallel.
         if profiles_to_import:
             note('Importing %d profiles with %d threads...' %
                  (len(profiles_to_import), multiprocessing.cpu_count()))
             TIMEOUT = 800
             try:
                 pool = multiprocessing.Pool()
def _parse_lit_output(self, data, only_test=False):
             # If we got this far, the result we were looking for didn't exist.
             raise RuntimeError("Result %s did not exist!" %
                                self.opts.single_result)
 
         # FIXME: Add more machine info!
         run_info = {
             'tag': 'nts'
         }
+        if self.opts.cc is not None:
             run_info.update(self._get_cc_info())
             run_info['run_order'] = run_info['inferred_run_order']
         if self.opts.run_order:
             run_info['run_order'] = self.opts.run_order
         
         machine_info = {
         }
         
         machine = lnt.testing.Machine(self.nick, machine_info)
         run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)

tests/runtest/test_suite.py

 # RUN:     --verbose \
 # RUN:     > %t.log 2> %t.err
 # RUN: FileCheck --check-prefix CHECK-USE-PERF-ALL < %t.err %s
 # CHECK-USE-PERF-ALL: Configuring with {
 # CHECK-USE-PERF-ALL:   TEST_SUITE_USE_PERF: 'ON'
 # CHECK-USE-PERF-ALL: --param profile=perf
 # CHECK-USE-PERF-ALL: Importing 1 profiles with
 # CHECK-USE-PERF-ALL: Profile /tmp/I/Do/Not/Exist.perf_data does not exist
+
+# Check --just-submit mode
+# RUN: lnt runtest test-suite \
+# RUN:     --just-submit=%S/Inputs/test-suite-cmake/fake-results.json \
+# RUN:     --run-order=123 \
+# RUN:     --verbose \
+# RUN:     > %t.log 2> %t.err
+# RUN: FileCheck --check-prefix CHECK-RESULTFILE --check-prefix CHECK-RESULTFILE-STDOUT < %t.log %s
+# RUN: FileCheck --check-prefix CHECK-RESULTFILE --check-prefix CHECK-RESULTFILE-STDERR < %t.err %s
+# Should not run any cmake/make/ninja/lit commands
+# CHECK-RESULTFILE-NOT: cmake
+# CHECK-RESULTFILE-NOT: make
+# CHECK-RESULTFILE-NOT: ninja
+# CHECK-RESULTFILE-NOT: lit
+# CHECK-RESULTFILE-NOT: clang
+# CHECK-RESULTFILE-STDERR: submitting result to dummy instance
+# CHECK-RESULTFILE-STDOUT: Import succeeded.
+# CHECK-RESULTFILE-STDOUT: PASS : 4