[lit] Add callback support for modifying test script

[ddcc]

Summary

Adds a callback field to TestConfig, and a "ShTest" callback in TestRunner to allow direct manipulation of command-line arguments.

This mechanism is proposed to be used by the Clang static analyzer in D28952 for checking if an additional constraint solver backend is available, and
if so, executing each test automatically with each constraint solver backend, without adding additional lines to each testcase.

docs/TestingGuide.rst

``%clang_cl``
    Invokes the CL-compatible Clang driver.
 
 ``%clangxx``
    Invokes the G++-compatible Clang driver.
 
 ``%clang_cc1``
    Invokes the Clang frontend.
 
+``%clang_analyze_cc1``
+   Invokes the Clang frontend with the static analyzer.
+
 ``%itanium_abi_triple``, ``%ms_abi_triple``
    These substitutions can be used to get the current target triple adjusted to
    the desired ABI. For example, if the test suite is running with the
    ``i686-pc-win32`` target, ``%itanium_abi_triple`` will expand to
    ``i686-pc-mingw32``. This allows a test to run with a specific ABI without
    constraining it to a specific triple.
 
 To add more substituations, look at ``test/lit.cfg`` or ``lit.local.cfg``.

utils/lit/lit/TestRunner.py

substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
                                              normalize_slashes=useExternalSh)
     script = applySubstitutions(script, substitutions)
 
     # Re-run failed tests up to test_retry_attempts times.
     attempts = 1
     if hasattr(test.config, 'test_retry_attempts'):
         attempts += test.config.test_retry_attempts
     for i in range(attempts):
+        if "ShTest" in test.config.callbacks:
+            script = test.config.callbacks["ShTest"](test, script)
         res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
         if res.code != Test.FAIL:
             break
     # If we had to run the test more than once, count it as a flaky pass. These
     # will be printed separately in the test summary.
     if i > 0 and res.code == Test.PASS:
         res.code = Test.FLAKYPASS
     return res

utils/lit/lit/TestingConfig.py

def load_from_path(self, path, litConfig):
                     path, traceback.format_exc()))
 
         self.finish(litConfig)
 
     def __init__(self, parent, name, suffixes, test_format,
                  environment, substitutions, unsupported,
                  test_exec_root, test_source_root, excludes,
                  available_features, pipefail, limit_to_features = [],
-                 is_early = False, parallelism_group = ""):
+                 is_early = False, parallelism_group = "", callbacks = {}):
         self.parent = parent
         self.name = str(name)
         self.suffixes = set(suffixes)
         self.test_format = test_format
         self.environment = dict(environment)
         self.substitutions = list(substitutions)
         self.unsupported = unsupported
         self.test_exec_root = test_exec_root
         self.test_source_root = test_source_root
         self.excludes = set(excludes)
         self.available_features = set(available_features)
         self.pipefail = pipefail
         # This list is used by TestRunner.py to restrict running only tests that
         # require one of the features in this list if this list is non-empty.
         # Configurations can set this list to restrict the set of tests to run.
         self.limit_to_features = set(limit_to_features)
         # Whether the suite should be tested early in a given run.
         self.is_early = bool(is_early)
         self.parallelism_group = parallelism_group
+        self.callbacks = callbacks
 
     def finish(self, litConfig):
         """finish() - Finish this config object, after loading is complete."""
 
         self.name = str(self.name)
         self.suffixes = set(self.suffixes)
         self.environment = dict(self.environment)
         self.substitutions = list(self.substitutions)
         if self.test_exec_root is not None:
             # FIXME: This should really only be suite in test suite config
             # files. Should we distinguish them?
             self.test_exec_root = str(self.test_exec_root)
         if self.test_source_root is not None:
             # FIXME: This should really only be suite in test suite config
             # files. Should we distinguish them?
             self.test_source_root = str(self.test_source_root)
         self.excludes = set(self.excludes)
+        self.callbacks = dict(self.callbacks)
 
     @property
     def root(self):
         """root attribute - The root configuration for the test suite."""
         if self.parent is None:
             return self
         else:
             return self.parent.root
 

utils/lit/tests/Inputs/shtest-callback/lit.cfg

+import lit.formats
+config.name = 'shtest-callback'
+config.substitutions = [('%callback1', '%callback2')]
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+config.target_triple = 'x86_64-unknown-unknown'

utils/lit/tests/Inputs/shtest-callback/lit.local.cfg

+import lit.TestRunner
+
+def callback(test, cmd):
+    return lit.TestRunner.applySubstitutions(cmd, [('%callback2', 'true')])
+
+config.callbacks["ShTest"] = callback
+

utils/lit/tests/Inputs/shtest-callback/shtest.txt

+# RUN: %callback1

utils/lit/tests/shtest-callback.py

+# Test callbacks are executed correctly.
+
+# RUN: %{lit} -j 1 -v %{inputs}/callback > %t.out
+# RUN: FileCheck < %t.out %s
+
+# CHECK: -- Testing:
+
+# CHECK: PASS: callback :: shtest.txt