[LNT] PGO and reference input support for SPEC

[anemet]

Summary

This adds the LNT parts for D14734.

Besides the new flags, we need to pass a few more things to the test module in order to invoke make inside. Also a few more shared utilities are added to the TestModule base class.

[cmatthews]

I am a little worried about the --pgo option misleading the user into thinking most things might be run with pgo. For now could we change it to --spec-with-pgo or something like that. If more of the test-suite supports PGO it would be fine to have a more general flag.

Besides that, LGTM.

[anemet]

In D14735#291094, @cmatthews wrote:

I am a little worried about the --pgo option misleading the user into thinking most things might be run with pgo. For now could we change it to --spec-with-pgo or something like that. If more of the test-suite supports PGO it would be fine to have a more general flag.

Makes sense, I will change it. Thanks for the reviews!

lnt/trunk/lnt/tests/nt.py

 import csv
 import os
 import platform
 import re
 import shutil
 import subprocess
 import sys
 import glob
 import time
 import traceback
 from datetime import datetime
 from optparse import OptionParser, OptionGroup
 import urllib2
 import shlex
 import pipes
+import resource
 
 import lnt.testing
 import lnt.testing.util.compilers
 import lnt.util.ImportData as ImportData
 
 from lnt.testing.util.commands import note, warning, fatal
 from lnt.testing.util.commands import capture, mkdir_p, which
 from lnt.testing.util.commands import resolve_command_path
class TestModule(object):
     """
 
     def __init__(self):
         self._log = None
 
     def main(self):
         raise NotImplementedError
 
-    def execute_test(self, options):
+    def execute_test(self, options, make_variables, config):
         raise RuntimeError("Abstract Method.")
 
-    def _execute_test(self, test_log, options):
+    def _execute_test(self, test_log, options, make_variables, config):
         self._log = test_log
+        self._user_time = config.test_time_stat == 'user'
         try:
-            return self.execute_test(options)
+            return self.execute_test(options, make_variables, config)
         finally:
             self._log = None
 
+    def call(self, args, **kwargs):
+        if kwargs.get('shell', False):
+            cmdstr = args
+        else:
+            cmdstr = ' '.join(args)
+
+        if 'cwd' in kwargs:
+            print >>self._log, "# In working dir: " + kwargs['cwd']
+        print >>self.log, cmdstr
+
+        self._log.flush()
+        p = subprocess.Popen(args, stdout=self._log, stderr=self._log, **kwargs)
+        return p.wait()
+
+    def get_time(self):
+        if self._user_time:
+            return resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
+        else:
+            return time.time()
+
     @property
     def log(self):
         """Get the test log output stream."""
         if self._log is None:
             raise ValueError("log() unavailable outside test execution")
         return self._log
 
 
def compute_run_make_variables(self):
         if self.test_benchmarking_only:
             make_variables['BENCHMARKING_ONLY'] = '1'
         if self.test_integrated_as:
             make_variables['TEST_INTEGRATED_AS'] = '1'
         if self.liblto_path:
             make_variables['LD_ENV_OVERRIDES'] = (
                 'env DYLD_LIBRARY_PATH=%s' % os.path.dirname(
                     self.liblto_path))
+        # If ref input is requested for SPEC, we wil run SPEC through its new test
+        # module so skip SPEC as part of NT.
+        if self.test_spec_ref:
+            make_variables['USE_SPEC_TEST_MODULE'] = '1'
 
         if self.threads > 1 or self.build_threads > 1:
             make_variables['ENABLE_PARALLEL_REPORT'] = '1'
 
         # Select the test style to use.
         if self.test_style == "simple":
             # We always use reference outputs with TEST=simple.
             make_variables['ENABLE_HASHED_PROGRAM_OUTPUT'] = '1'
def scan_for_test_modules(config):
     # users to just checkout them out elsewhere and link them into their LLVM
     # test-suite source tree.
     for dirpath,dirnames,filenames in os.walk(test_modules_path,
                                               followlinks = True):
         # Ignore the example tests, unless requested.
         if not config.include_test_examples and 'Examples' in dirnames:
             dirnames.remove('Examples')
 
+        # Only use the SPEC test module if the ref input was requested.
+        if not config.test_spec_ref and re.search('/spec', dirpath):
+            continue
+
         # Check if this directory defines a test module.
         if 'TestModule' not in filenames:
             continue
 
         # If so, don't traverse any lower.
         del dirnames[:]
 
         # Add to the list of test modules.
while p.poll() is None:
         global_log.write(l)
 
     global_log.close()
 
   return p.wait()
 
 # FIXME: Support duplicate logfiles to global directory.
 def execute_test_modules(test_log, test_modules, test_module_variables,
-                         basedir, config):
+                         basedir, config, make_variables):
     # For now, we don't execute these in parallel, but we do forward the
     # parallel build options to the test.
     test_modules.sort()
 
     print >>sys.stderr, '%s: executing test modules' % (timestamp(),)
     results = []
     for name in test_modules:
         # First, load the test module file.
         locals = globals = {}
         test_path = os.path.join(config.test_suite_root, 'LNTBased', name)
+        # This is where shared code between test modules should go.
+        sys.path.append(os.path.join(config.test_suite_root, 'LNTBased/lib'))
         test_obj_path = os.path.join(basedir, 'LNTBased', name)
         module_path = os.path.join(test_path, 'TestModule')
         module_file = open(module_path)
         try:
             exec module_file in locals, globals
         except:
             info = traceback.format_exc()
             fatal("unable to import test module: %r\n%s" % (
                     module_path, info))
 
         # Lookup and instantiate the test class.
         test_class = globals.get('test_class')
         if test_class is None:
             fatal("no 'test_class' global in import test module: %r" % (
                     module_path,))
         try:
             test_instance = test_class()
         except:
-            fatal("unable to instantiate test class for: %r" % module_path)
+            info = traceback.format_exc()
+            fatal("unable to instantiate test class for: %r\n%s" % (
+                    module_path, info))
 
         if not isinstance(test_instance, TestModule):
             fatal("invalid test class (expected lnt.tests.nt.TestModule "
                   "subclass) for: %r" % module_path)
 
         # Create the per test variables, and ensure the output directory exists.
         variables = test_module_variables.copy()
         variables['MODULENAME'] = name
         variables['SRCROOT'] = test_path
         variables['OBJROOT'] = test_obj_path
         mkdir_p(test_obj_path)
 
         # Execute the tests.
         try:
-            test_samples = test_instance._execute_test(test_log, variables)
+            test_samples = test_instance._execute_test(test_log, variables, make_variables, config)
         except:
             info = traceback.format_exc()
             fatal("exception executing tests for: %r\n%s" % (
                     module_path, info))
 
         # Check that the test samples are in the expected format.
         is_ok = True
         try:
# Always blow away any existing report.
     run_nightly_test = (config.only_test is None or
                         not config.only_test.startswith("LNTBased"))
     if run_nightly_test:
         execute_nt_tests(test_log, make_variables, basedir, config)
 
     # Run the extension test modules, if needed.
     test_module_results = execute_test_modules(test_log, test_modules,
                                                test_module_variables, basedir,
-                                               config)
+                                               config, make_variables)
     test_log.close()
 
     end_time = timestamp()
 
     # Load the nightly test samples.
     if config.test_style == "simple":
         test_namespace = 'nts'
     else:
def run_test(self, name, args):
                          action="append", type=str, default=[], metavar="FLAG")
         group.add_option("", "--cflags", dest="cflag_string",
                          help="Additional flags to set in TARGET_FLAGS, space separated string. "
                          "These flags are appended after *all* the individual --cflag arguments.",
                          type=str, default='', metavar="FLAG")
         group.add_option("", "--mllvm", dest="mllvm",
                          help="Add -mllvm FLAG to TARGET_FLAGS",
                          action="append", type=str, default=[], metavar="FLAG")
+        group.add_option("", "--spec-with-pgo", help="Use PGO with SPEC",
+                         action="store_true")
         parser.add_option_group(group)
 
         group = OptionGroup(parser, "Test Selection")
         group.add_option("", "--build-mode", dest="build_mode", metavar="NAME",
                          help="Select the LLVM build mode to use [%default]",
                          type=str, action="store", default='Release+Asserts')
 
         group.add_option("", "--simple", dest="test_simple",
def run_test(self, name, args):
                          action="store_false", default=True)
 
         group.add_option("", "--small", dest="test_small",
                          help="Use smaller test inputs and disable large tests",
                          action="store_true", default=False)
         group.add_option("", "--large", dest="test_large",
                          help="Use larger test inputs",
                          action="store_true", default=False)
+        group.add_option("", "--spec-with-ref", dest="test_spec_ref",
+                         help="Use reference test inputs for SPEC.  "
+                         "This is currently experimental",
+                         action="store_true", default=False)
         group.add_option("", "--benchmarking-only", dest="test_benchmarking_only",
                          help="Benchmarking-only mode",
                          action="store_true", default=False)
 
         group.add_option("", "--only-test", dest="only_test", metavar="PATH",
                          help="Only run tests under PATH",
                          type=str, default=None)
         group.add_option("", "--include-test-examples",
def run_test(self, name, args):
             if opts.remote_user is None:
                 parser.error('--remote-user is required with --remote')
         else:
             if opts.remote_port is not None:
                 parser.error('--remote is required with --remote-port')
             if opts.remote_user is not None:
                 parser.error('--remote is required with --remote-user')
 
+        if opts.test_spec_ref and opts.remote:
+            parser.error('--remote with --spec-with-ref is not yet supported')
+
+        if opts.spec_with_pgo and not opts.test_spec_ref:
+            parser.error('--spec-with-pgo is only supported with --spec-with-ref')
+
         # libLTO should exist, if given.
         if opts.liblto_path:
             if not os.path.exists(opts.liblto_path):
                 parser.error('invalid --liblto-path argument %r' % (
                         opts.liblto_path,))
 
         # Support disabling test suite externals separately from providing path.
         if not opts.test_externals: