diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -410,11 +410,12 @@ configuration parameters --- for example, to change the test format, or the suffixes which identify test files. -PRE-DEFINED SUBSTITUTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~ +SUBSTITUTIONS +~~~~~~~~~~~~~ -:program:`lit` provides various patterns that can be used with the RUN command. -These are defined in TestRunner.py. The base set of substitutions are: +:program:`lit` allows patterns to be substituted inside RUN commands. It also +provides the following base set of substitutions, which are defined in +TestRunner.py: ======================= ============== Macro Substitution @@ -453,6 +454,14 @@ further substitution patterns can be defined by each test module. See the modules :ref:`local-configuration-files`. +By default, substitutions are expanded exactly once, so that if e.g. a +substitution ``%build`` is defined in top of another substitution ``%cxx``, +``%build`` will expand to ``%cxx`` textually, not to what ``%cxx`` expands to. +However, if the ``recursiveExpansionLimit`` property of the ``LitConfig`` is +set to a non-negative integer, substitutions will be expanded recursively until +that limit is reached. It is an error if the limit is reached and expanding +substitutions again would yield a different result. + More detailed information on substitutions can be found in the :doc:`../TestingGuide`. diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py --- a/llvm/utils/lit/lit/LitConfig.py +++ b/llvm/utils/lit/lit/LitConfig.py @@ -66,6 +66,19 @@ self.maxIndividualTestTime = maxIndividualTestTime self.parallelism_groups = parallelism_groups self.echo_all_commands = echo_all_commands + self._recursiveExpansionLimit = None + + @property + def recursiveExpansionLimit(self): + return self._recursiveExpansionLimit + + @recursiveExpansionLimit.setter + def recursiveExpansionLimit(self, value): + if value is not None and not isinstance(value, int): + self.fatal('recursiveExpansionLimit must be either None or an integer (got <{}>)'.format(value)) + if isinstance(value, int) and value < 0: + self.fatal('recursiveExpansionLimit must be a non-negative integer (got <{}>)'.format(value)) + self._recursiveExpansionLimit = value @property def maxIndividualTestTime(self): diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1147,10 +1147,18 @@ def _caching_re_compile(r): return re.compile(r) -def applySubstitutions(script, substitutions): - """Apply substitutions to the script. Allow full regular expression syntax. +def applySubstitutions(script, substitutions, recursion_limit=None): + """ + Apply substitutions to the script. Allow full regular expression syntax. Replace each matching occurrence of regular expression pattern a with - substitution b in line ln.""" + substitution b in line ln. + + If a substitution expands into another substitution, it is expanded + recursively until the line has no more expandable substitutions. If + the line can still can be substituted after being substituted + `recursion_limit` times, it is an error. If the `recursion_limit` is + `None` (the default), no recursive substitution is performed at all. + """ def processLine(ln): # Apply substitutions for a,b in substitutions: @@ -1167,9 +1175,28 @@ # Strip the trailing newline and any extra whitespace. return ln.strip() + + def processLineToFixedPoint(ln): + assert isinstance(recursion_limit, int) and recursion_limit >= 0 + origLine = ln + steps = 0 + processed = processLine(ln) + while processed != ln and steps < recursion_limit: + ln = processed + processed = processLine(ln) + steps += 1 + + if processed != ln: + raise ValueError("Recursive substitution of '%s' did not complete " + "in the provided recursion limit (%s)" % \ + (origLine, recursion_limit)) + + return processed + # Note Python 3 map() gives an iterator rather than a list so explicitly # convert to list before returning. - return list(map(processLine, script)) + process = processLine if recursion_limit is None else processLineToFixedPoint + return list(map(process, script)) class ParserKind(object): @@ -1506,7 +1533,8 @@ substitutions = list(extra_substitutions) substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=useExternalSh) - script = applySubstitutions(script, substitutions) + script = applySubstitutions(script, substitutions, + recursion_limit=litConfig.recursiveExpansionLimit) # Re-run failed tests up to test.allowed_retries times. attempts = test.allowed_retries + 1 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg @@ -0,0 +1,10 @@ +import lit.formats +config.name = 'does-not-substitute-no-limit' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), + ("%rec5", "%rec4")] diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py @@ -0,0 +1 @@ +# RUN: echo %rec5 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg @@ -0,0 +1,12 @@ +import lit.formats +config.name = 'does-not-substitute-within-limit' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), + ("%rec5", "%rec4")] + +lit_config.recursiveExpansionLimit = 2 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py @@ -0,0 +1 @@ +# RUN: echo %rec5 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +config.name = 'negative-integer' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +lit_config.recursiveExpansionLimit = -4 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +config.name = 'not-an-integer' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +lit_config.recursiveExpansionLimit = "not-an-integer" diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +config.name = 'set-to-none' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +lit_config.recursiveExpansionLimit = None diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py @@ -0,0 +1 @@ +# RUN: true diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg @@ -0,0 +1,12 @@ +import lit.formats +config.name = 'substitutes-within-limit' +config.suffixes = ['.py'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None + +config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), + ("%rec5", "%rec4")] + +lit_config.recursiveExpansionLimit = 5 diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py @@ -0,0 +1 @@ +# RUN: echo %rec5 diff --git a/llvm/utils/lit/tests/shtest-recursive-substitution.py b/llvm/utils/lit/tests/shtest-recursive-substitution.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/shtest-recursive-substitution.py @@ -0,0 +1,23 @@ +# Check that the config.recursiveExpansionLimit is picked up and will cause +# lit substitutions to be expanded recursively. + +# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/substitutes-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST1 %s +# CHECK-TEST1: PASS: substitutes-within-limit :: test.py +# CHECK-TEST1: $ "echo" "STOP" + +# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST2 %s +# CHECK-TEST2: UNRESOLVED: does-not-substitute-within-limit :: test.py +# CHECK-TEST2: ValueError: Recursive substitution of + +# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-no-limit --show-all | FileCheck --check-prefix=CHECK-TEST3 %s +# CHECK-TEST3: PASS: does-not-substitute-no-limit :: test.py +# CHECK-TEST3: $ "echo" "%rec4" + +# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/not-an-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST4 %s +# CHECK-TEST4: recursiveExpansionLimit must be either None or an integer + +# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/negative-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST5 %s +# CHECK-TEST5: recursiveExpansionLimit must be a non-negative integer + +# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s +# CHECK-TEST6: PASS: set-to-none :: test.py diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py --- a/llvm/utils/lit/tests/unit/TestRunner.py +++ b/llvm/utils/lit/tests/unit/TestRunner.py @@ -199,6 +199,74 @@ except BaseException as e: self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e) +class TestApplySubtitutions(unittest.TestCase): + def test_simple(self): + script = ["echo %bar"] + substitutions = [("%bar", "hello")] + result = lit.TestRunner.applySubstitutions(script, substitutions) + self.assertEqual(result, ["echo hello"]) + + def test_multiple_substitutions(self): + script = ["echo %bar %baz"] + substitutions = [("%bar", "hello"), + ("%baz", "world"), + ("%useless", "shouldnt expand")] + result = lit.TestRunner.applySubstitutions(script, substitutions) + self.assertEqual(result, ["echo hello world"]) + + def test_multiple_script_lines(self): + script = ["%cxx %compile_flags -c -o %t.o", + "%cxx %link_flags %t.o -o %t.exe"] + substitutions = [("%cxx", "clang++"), + ("%compile_flags", "-std=c++11 -O3"), + ("%link_flags", "-lc++")] + result = lit.TestRunner.applySubstitutions(script, substitutions) + self.assertEqual(result, ["clang++ -std=c++11 -O3 -c -o %t.o", + "clang++ -lc++ %t.o -o %t.exe"]) + + def test_recursive_substitution_real(self): + script = ["%build %s"] + substitutions = [("%cxx", "clang++"), + ("%compile_flags", "-std=c++11 -O3"), + ("%link_flags", "-lc++"), + ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe")] + result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=3) + self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"]) + + def test_recursive_substitution_limit(self): + script = ["%rec5"] + # Make sure the substitutions are not in an order where the global + # substitution would appear to be recursive just because they are + # processed in the right order. + substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")] + for limit in [5, 6, 7]: + result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit) + self.assertEqual(result, ["STOP"]) + + def test_recursive_substitution_limit_exceeded(self): + script = ["%rec5"] + substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")] + for limit in [0, 1, 2, 3, 4]: + try: + lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit) + self.fail("applySubstitutions should have raised an exception") + except ValueError: + pass + + def test_recursive_substitution_invalid_value(self): + script = ["%rec5"] + substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"), + ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")] + for limit in [-1, -2, -3, "foo"]: + try: + lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit) + self.fail("applySubstitutions should have raised an exception") + except AssertionError: + pass + + if __name__ == '__main__': TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests() unittest.main(verbosity=2)