diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -556,6 +556,7 @@ Otherwise, %t but with a single leading ``/`` removed. %:T On Windows, %/T but a ``:`` is removed if its the second character. Otherwise, %T but with a single leading ``/`` removed. + %{for-each-file } Invokes RUN command for each file in ````. Accepts wildcards. ======================= ============== Other substitutions are provided that are variations on this base set and diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -279,6 +279,8 @@ * Made significant changes to JSON output format of `llvm-readobj`/`llvm-readelf` to improve correctness and clarity. +* llvm-lit now accepts %{for-each-file} in RUN commands. + Changes to LLDB --------------------------------- diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1529,9 +1529,13 @@ # We use #_MARKER_# to hide %% while we do the other substitutions. def escapePercents(ln): + if isinstance(ln, list): + return [_caching_re_compile("%%").sub("#_MARKER_#", line) for line in ln] return _caching_re_compile("%%").sub("#_MARKER_#", ln) def unescapePercents(ln): + if isinstance(ln, list): + return [_caching_re_compile("#_MARKER_#").sub("%", line) for line in ln] return _caching_re_compile("#_MARKER_#").sub("%", ln) def substituteIfElse(ln): @@ -1623,6 +1627,45 @@ assert len(ln) == 0 return result + def substituteForEachFile(ln): + from glob import iglob + from pathlib import Path + + if "%{for-each-file" not in ln: + return [ln] + + pattern = re.compile(r"%{for-each-file ([^}]+)}") + matches = list(pattern.finditer(ln)) + if len(matches) == 0: + raise ValueError("%{for-each-file} expects a path, but it is not specified") + elif len(matches) > 1: + # FIXME: here we make sure there is only one %{for-each-file} + # to expand, but we can support multiple via cartesian product. + # It doesn't seem immediately useful, so we are keeping this + # function simple. + raise ValueError( + "multiple %{for-each-file} per directive are not supported yet" + ) + match = matches[0] + path = Path(match[1]) + if not path.is_absolute(): + raise ValueError( + "%{{for-each-file}} expects an absolute path," + " but '{}' is not absolute".format(str(path)) + ) + path.resolve() + if path.is_dir() and "*" not in str(path): + path = path / "*" + + lines = [] + has_matches = False + for file in iglob(str(path), recursive=True): + has_matches = True + lines.append(pattern.sub(file, ln)) + if not has_matches: + raise ValueError("%{{for-each-file {}}} doesn't match anything".format(str(path))) + return lines + def processLine(ln): # Apply substitutions ln = substituteIfElse(escapePercents(ln)) @@ -1660,7 +1703,7 @@ return processed process = processLine if recursion_limit is None else processLineToFixedPoint - output = [] + output_stage1 = [] for directive in script: if isinstance(directive, SubstDirective): directive.adjust_substitutions(substitutions) @@ -1671,9 +1714,13 @@ # Can come from preamble_commands. assert isinstance(directive, str) line = directive - output.append(unescapePercents(process(line))) + output_stage1.append(process(line)) + + output_stage2 = [] + for directive_str in output_stage1: + output_stage2.extend(unescapePercents(substituteForEachFile(directive_str))) - return output + return output_stage2 class ParserKind(object): diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-multiple.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-multiple.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-multiple.txt @@ -0,0 +1,2 @@ +# RUN: cat %{for-each-file %S} %{for-each-file %S} +# CHECK: ValueError: multiple %{for-each-file} per directive are not supported yet diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg-2.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg-2.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg-2.txt @@ -0,0 +1,2 @@ +# RUN: cat %{for-each-file } +# CHECK: ValueError: %{for-each-file} expects a path, but it is not specified diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-arg.txt @@ -0,0 +1,2 @@ +# RUN: cat %{for-each-file} +# CHECK: ValueError: %{for-each-file} expects a path, but it is not specified diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-match.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-match.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-no-match.txt @@ -0,0 +1,2 @@ +# RUN: cat %{for-each-file %S/inputs/*.cpp} +# CHECK: ValueError: %{for-each-file {{.*}}/inputs/*.cpp} doesn't match anything diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-rel-path.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-rel-path.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/error-rel-path.txt @@ -0,0 +1,2 @@ +# RUN: cat %{for-each-file relative_path} +# CHECK: ValueError: %{for-each-file} expects an absolute path, but 'relative_path' is not absolute diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/for-each-file.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/for-each-file.txt new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/for-each-file.txt @@ -0,0 +1,15 @@ +# RUN: cat %{for-each-file %S/inputs} +# RUN: cat %{for-each-file %S/inputs/} +# RUN: cat %{for-each-file %S/inputs/*} +# RUN: cat %{for-each-file %S/inputs/*.txt} + +# CHECK: Script +# CHECK-NEXT: -- +# CHECK-NEXT: {{.*}} at line 1'; cat {{.*}}a.txt +# CHECK-NEXT: {{.*}} at line 1'; cat {{.*}}b.txt +# CHECK-NEXT: {{.*}} at line 2'; cat {{.*}}a.txt +# CHECK-NEXT: {{.*}} at line 2'; cat {{.*}}b.txt +# CHECK-NEXT: {{.*}} at line 3'; cat {{.*}}a.txt +# CHECK-NEXT: {{.*}} at line 3'; cat {{.*}}b.txt +# CHECK-NEXT: {{.*}} at line 4'; cat {{.*}}a.txt +# CHECK-NEXT: {{.*}} at line 4'; cat {{.*}}b.txt diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/inputs/a.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/inputs/a.txt new file mode 100644 diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/inputs/b.txt b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/inputs/b.txt new file mode 100644 diff --git a/llvm/utils/lit/tests/Inputs/shtest-for-each-file/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/lit.cfg new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/shtest-for-each-file/lit.cfg @@ -0,0 +1,5 @@ +import lit.formats + +config.name = "shtest-for-each-file" +config.suffixes = [".txt"] +config.test_format = lit.formats.ShTest() diff --git a/llvm/utils/lit/tests/shtest-for-each-file.py b/llvm/utils/lit/tests/shtest-for-each-file.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/shtest-for-each-file.py @@ -0,0 +1,7 @@ +# RUN: %{lit} -a -v %{inputs}/shtest-for-each-file/for-each-file.txt | FileCheck %{inputs}/shtest-for-each-file/for-each-file.txt + +# RUN: not %{lit} -a -v %{inputs}/shtest-for-each-file/error-multiple.txt | FileCheck %{inputs}/shtest-for-each-file/error-multiple.txt +# RUN: not %{lit} -a -v %{inputs}/shtest-for-each-file/error-no-arg.txt | FileCheck %{inputs}/shtest-for-each-file/error-no-arg.txt +# RUN: not %{lit} -a -v %{inputs}/shtest-for-each-file/error-no-arg-2.txt | FileCheck %{inputs}/shtest-for-each-file/error-no-arg-2.txt +# RUN: not %{lit} -a -v %{inputs}/shtest-for-each-file/error-no-match.txt | FileCheck %{inputs}/shtest-for-each-file/error-no-match.txt +# RUN: not %{lit} -a -v %{inputs}/shtest-for-each-file/error-rel-path.txt | FileCheck %{inputs}/shtest-for-each-file/error-rel-path.txt