diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -27,8 +27,9 @@ fail. By default :program:`lit` will use a succinct progress display and will only -print summary information for test failures. See :ref:`output-options` for -options controlling the :program:`lit` progress display and output. +print summary information for test failures, such as which exact line failed. +See :ref:`output-options` for options controlling the :program:`lit` +progress display and output. :program:`lit` also includes a number of options for controlling how tests are executed (specific features may depend on the particular test format). See @@ -88,8 +89,6 @@ Show more information on test failures, for example the entire test output instead of just the test result. -.. option:: -vv, --echo-all-commands - Echo all commands to stdout, as they are being executed. This can be valuable for debugging test failures, as the last echoed command will be the one which has failed. @@ -97,8 +96,10 @@ with argument ``'RUN: at line N'`` before each command pipeline, and this option also causes those no-op commands to be echoed to stdout to help you locate the source line of the failed command. - This option implies ``--verbose``. +.. option:: -vv, --echo-all-commands + + Alias for ``-v``/``--verbose`` (for backwards compatibility). .. option:: -a, --show-all Show more information about all tests, for example the entire test diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py --- a/llvm/utils/lit/lit/LitConfig.py +++ b/llvm/utils/lit/lit/LitConfig.py @@ -6,6 +6,7 @@ import lit.Test import lit.formats +import lit.OutputSettings as OutputSettings import lit.TestingConfig import lit.util @@ -26,7 +27,9 @@ params, config_prefix = None, maxIndividualTestTime = 0, parallelism_groups = {}, - echo_all_commands = False): + echo_all_commands=False, + script_output_style=OutputSettings.NO_SCRIPT, + command_output_style=OutputSettings.ONLY_FAILING_COMMAND): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -66,6 +69,8 @@ self.maxIndividualTestTime = maxIndividualTestTime self.parallelism_groups = parallelism_groups self.echo_all_commands = echo_all_commands + self.script_output_style = script_output_style + self.command_output_style = command_output_style @property def maxIndividualTestTime(self): diff --git a/llvm/utils/lit/lit/OutputSettings.py b/llvm/utils/lit/lit/OutputSettings.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/lit/OutputSettings.py @@ -0,0 +1,66 @@ +# TODO(python3): Could be an enum +class ScriptOutputStyle: + def __init__(self, name): + self.name = name + + # For print-debugging. + def __str__(self): + return '%s(%r)' % (self.__class__.__name__, self.name) + +NO_SCRIPT = ScriptOutputStyle("None") +FULL_SCRIPT = ScriptOutputStyle("Full") + +# Consider a script +# +# ``` +# RUN: true +# RUN: false +# RUN: true +# ``` +# +# FULL_SCRIPT will show the whole script. + +# TODO(python3): Could be an enum +class CommandOutputStyle: + def __init__(self, name): + self.name = name + + # For print-debugging. + def __str__(self): + return '%s(%r)' % (self.__class__.__name__, self.name) + +NO_COMMAND = CommandOutputStyle("None") +ONLY_FAILING_COMMAND = CommandOutputStyle("OnlyFailing") +UP_TO_AND_INCLUDING_FAILING_COMMAND = CommandOutputStyle("UpToAndIncluding") + +# Consider a script +# +# ``` +# RUN: true +# RUN: false +# RUN: true +# ``` +# +# ONLY_FAILING_COMMAND will show something like +# +# ``` +# Command output (stderr, truncated) +# -- +# + : 'RUN: at line 2' +# + false +# -- +# NOTE: The failure might depend on preceding RUN lines. +# Use --verbose to see preceding RUN lines and outputs. +# ``` +# +# UP_TO_AND_INCLUDING_FAILING_COMMAND will show something like +# +# ``` +# Command Output (stderr): +# -- +# + : 'RUN: at line 1' +# + true +# + : 'RUN: at line 2' +# + false +# -- +# ``` diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -17,6 +17,7 @@ except ImportError: from io import StringIO +import lit.OutputSettings as OutputSettings from lit.ShCommands import GlobItem, Command import lit.ShUtil as ShUtil import lit.Test as Test @@ -1495,6 +1496,133 @@ return script +def locate_last_run_line(output_str): + """ + Try to locate the last run line in ``output_str`` using heuristics. + + Returns a pair of: + - The index in ``output_str`` pointing to immediately after the preceding + newline, i.e. the start of the RUN line, before any shell-specific + prefix. + - The matched substring itself, including the number at the end, + starting with 'RUN', skipping the shell-specific prefix. + + For example, if ``output_str`` is ``"+ HI\n+ RUN: at line 10"``, + the index will be 5 and the substring will be ``"RUN: at line 10"``. + + Returns (-1, None) on failure. + """ + # We try to be clever; rather than directly finding the last + # 'RUN: at line', we try to search from the beginning, create a regex, + # which hopefully takes into account any shell-specific prefix, + # and then uses that regex to search from the end. + # Searching backwards directly can lead to erroneous reporting + # when the command output itself has a 'RUN: at line' due to something + # else (possibly because we are testing ``lit`` itself). + # + # However, this is not perfect either :(. It will fail if a shell emits + # varying prefixes for commands on 'set +x', such as line numbers or + # timestamps. So we try to fallback to the simpler technique if applicable. + + error = (-1, None) + run_str = "RUN: at line" + first_run_str_start = output_str.find(run_str) + if first_run_str_start == -1: + return error + # Use the first RUN line to create a regex for searching. + newline_index = output_str[:first_run_str_start].rfind('\n') + first_run_line_start = 0 if newline_index == -1 else newline_index + 1 + first_run_str_end = first_run_str_start + len(run_str) + run_line_regex = ( + '(' + + re.escape(output_str[first_run_line_start:first_run_str_end]) + + ' [0-9]+)' + ) + + run_lines = list(re.finditer(run_line_regex, output_str)) + if len(run_lines) == 0: + # impossible; we should've at least gotten the first run line... + return error + + if len(run_lines) == 1: + # We must've gotten our original string. That's not helpful. :-/ + # Try using the simple strategy to make things work with shells that + # use a varying prefix for 'set +x'. + # (N.B. I don't know of any shells that actually do this, but it's + # not inconceivable that a CI machine would, say, print out timestamps + # with 'set +x') + simple_run_line_regex = '(RUN: at line [0-9]+)' + simple_run_lines = list(re.finditer(simple_run_line_regex, output_str)) + if len(simple_run_lines) == 0: + # impossible; we should've at least gotten the first run line... + return error + elif len(simple_run_lines) == 1: + # we must've hit the line we found earlier, don't try anything + pass + else: + # Assume that we have a shell with a varying prefix and use + # the results of the simple matching. + # However, this creates a problem when: + # 1. The very first RUN line failed and it has 'RUN: at line'. + # 2. A shell has varying prefixes and the output of the last command + # has 'RUN: at line'. + # Can't really help these situations. :( + last_run_line = simple_run_lines[-1] + newline_index = output_str[:last_run_line.start()].rfind('\n') + start = 0 if newline_index == -1 else (newline_index + 1) + return (start, last_run_line.group(0)) + + last_run_line = run_lines[-1] + # trim the shell-specific prefix + delta = first_run_str_start - first_run_line_start + last_run_line_str = last_run_line.group(0)[delta:] + return (last_run_line.start(), last_run_line_str) + +def make_script_output(lit_config, script_lines, exit_code): + def make_output(script_display_lines): + return ("""Script:\n--\n%s\n--\nExit Code: %d\n""" + % ('\n'.join(script_display_lines), exit_code)) + + def default_output(): + return make_output(script_lines) + + if lit_config.script_output_style == OutputSettings.NO_SCRIPT: + return "" + + assert(lit_config.script_output_style == OutputSettings.FULL_SCRIPT) + return default_output() + +def make_command_output(lit_config, cmd_output, stream_name, test_status): + def make_output(output_str, is_truncated=False): + if is_truncated: + format_str = """Command Output (%s, truncated):\n--\n%s\n--\n""" + format_str += ( + "(NOTE: The failure may depend on previous RUN lines.\n" + " Use --verbose to see previous RUN lines and outputs.)\n" + ) + else: + format_str = """Command Output (%s):\n--\n%s\n--\n""" + return format_str % (stream_name, output_str) + + def default_output(): + return make_output(cmd_output) + + if lit_config.command_output_style == OutputSettings.NO_COMMAND: + return "" + elif not test_status.isFailure: + return default_output() + + line_start, run_line_str = locate_last_run_line(cmd_output) + # maybe there was an error, or maybe we are not truncating anything + if run_line_str is None or line_start == 0: + return default_output() + + if lit_config.command_output_style == OutputSettings.ONLY_FAILING_COMMAND: + return make_output(cmd_output[line_start:], is_truncated=True) + + assert(lit_config.command_output_style + == OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND) + return make_output(cmd_output, is_truncated=False) def _runShTest(test, litConfig, useExternalSh, script, tmpBase): def runOnce(execdir): @@ -1536,8 +1664,7 @@ status = Test.FLAKYPASS # Form the output log. - output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % ( - '\n'.join(script), exitCode) + output = make_script_output(litConfig, script, exitCode) if timeoutInfo is not None: output += """Timeout: %s\n""" % (timeoutInfo,) @@ -1545,9 +1672,9 @@ # Append the outputs, if present. if out: - output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) + output += make_command_output(litConfig, out, "stdout", status) if err: - output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) + output += make_command_output(litConfig, err, "stderr", status) return lit.Test.Result(status, output) diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -3,6 +3,7 @@ import shlex import sys +import lit.OutputSettings as OutputSettings import lit.reports import lit.util @@ -47,14 +48,15 @@ " unless --no-progress-bar is specified.", action="store_true") format_group.add_argument("-v", "--verbose", - dest="showOutput", - help="Show test output for failures", + help="Show test output for failures." + " The last command under 'Output' is the failing one." + " All substituted commands are shown under 'Script'," + " including any commands after the failing command.", action="store_true") format_group.add_argument("-vv", "--echo-all-commands", - dest="echoAllCommands", - action="store_true", - help="Echo all commands as they are executed to stdout. In case of " - "failure, last command shown will be the failing one.") + help="Alias for --verbose.", + dest="verbose", + action="store_true") format_group.add_argument("-a", "--show-all", dest="showAllOutput", help="Display all commandlines and output", @@ -168,9 +170,30 @@ args = sys.argv[1:] + env_args opts = parser.parse_args(args) - # Validate command line options - if opts.echoAllCommands: - opts.showOutput = True + # default output settings + opts.show_output_on_failure = True + opts.script_output_style = OutputSettings.NO_SCRIPT + opts.command_output_style = OutputSettings.ONLY_FAILING_COMMAND + + if opts.quiet: + opts.show_output_on_failure = False + opts.command_output_style = OutputSettings.NO_COMMAND + elif opts.verbose or opts.showAllOutput: + opts.script_output_style = OutputSettings.FULL_SCRIPT + opts.command_output_style = ( + OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND + ) + + # If we call out to an external shell, we ask for all output, + # regardless of whether we're going to display all commands that were run, + # or only the last command which failed. + cmd_output_style = opts.command_output_style + opts.echo_all_commands = ( + cmd_output_style in { + OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND, + OutputSettings.ONLY_FAILING_COMMAND + } + ) # TODO(python3): Could be enum if opts.shuffle: diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py --- a/llvm/utils/lit/lit/display.py +++ b/llvm/utils/lit/lit/display.py @@ -69,7 +69,7 @@ self.completed, self.tests)) # Show the test failure output, if requested. - if (test.isFailure() and self.opts.showOutput) or \ + if (test.isFailure() and self.opts.show_output_on_failure) or \ self.opts.showAllOutput: if test.isFailure(): print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(), diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -37,7 +37,9 @@ isWindows=is_windows, params=params, config_prefix=opts.configPrefix, - echo_all_commands=opts.echoAllCommands) + echo_all_commands=opts.echo_all_commands, + script_output_style=opts.script_output_style, + command_output_style=opts.command_output_style) discovered_tests = lit.discovery.find_tests_for_inputs(lit_config, opts.test_paths) if not discovered_tests: diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py --- a/llvm/utils/lit/tests/shtest-format.py +++ b/llvm/utils/lit/tests/shtest-format.py @@ -17,7 +17,12 @@ # CHECK-NEXT: line 2: failed test output on stdout # CHECK: Command Output (stderr): # CHECK-NEXT: -- -# CHECK-NEXT: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}} +# CHECK-NEXT: 'RUN: at line 3' +# +# Skipping some lines... +# +# CHECK: 'RUN: at line 5' +# CHECK: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}} # CHECK: -- # CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py --- a/llvm/utils/lit/tests/shtest-run-at-line.py +++ b/llvm/utils/lit/tests/shtest-run-at-line.py @@ -1,70 +1,132 @@ -# Check that -vv makes the line number of the failing RUN command clear. -# (-v is actually sufficient in the case of the internal shell.) +# Check that the default terse output shows only the failing +# line in the output, and doesn't show the script. # -# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out -# RUN: FileCheck --input-file %t.out %s +# RUN: not %{lit} -j 1 %{inputs}/shtest-run-at-line > %t-terse.out +# RUN: FileCheck --input-file %t-terse.out %s --check-prefix=TERSE + +# Check that --verbose outputs both Script and Command Output. +# +# RUN: not %{lit} -j 1 --verbose %{inputs}/shtest-run-at-line > %t-verbose.out +# RUN: FileCheck --input-file %t-verbose.out %s --check-prefix=VERBOSE # # END. +# In the case of the external shell, we check for only RUN lines in stderr in +# case some shell implementations format "set -x" output differently. -# CHECK: Testing: 4 tests +################################################################################ +# Checking lines for terse output +# TERSE-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt -# In the case of the external shell, we check for only RUN lines in stderr in -# case some shell implementations format "set -x" output differently. +# TERSE-NOT: Script: -# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt - -# CHECK: Script: -# CHECK: RUN: at line 4{{.*}} true -# CHECK-NEXT: RUN: at line 5{{.*}} false -# CHECK-NEXT: RUN: at line 6{{.*}} true - -# CHECK: RUN: at line 4 -# CHECK: RUN: at line 5 -# CHECK-NOT: RUN - -# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt - -# CHECK: Script: -# CHECK: RUN: at line 4{{.*}} echo 'foo bar' | FileCheck -# CHECK-NEXT: RUN: at line 6{{.*}} echo 'foo baz' | FileCheck -# CHECK-NEXT: RUN: at line 9{{.*}} echo 'foo bar' | FileCheck - -# CHECK: RUN: at line 4 -# CHECK: RUN: at line 6 -# CHECK-NOT: RUN - - -# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt - -# CHECK: Script: -# CHECK: : 'RUN: at line 1'; true -# CHECK-NEXT: : 'RUN: at line 2'; false -# CHECK-NEXT: : 'RUN: at line 3'; true - -# CHECK: Command Output (stdout) -# CHECK: $ ":" "RUN: at line 1" -# CHECK-NEXT: $ "true" -# CHECK-NEXT: $ ":" "RUN: at line 2" -# CHECK-NEXT: $ "false" -# CHECK-NOT: RUN - -# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt - -# CHECK: Script: -# CHECK: : 'RUN: at line 1'; : first line continued to second line -# CHECK-NEXT: : 'RUN: at line 3'; echo 'foo bar' | FileCheck -# CHECK-NEXT: : 'RUN: at line 5'; echo 'foo baz' | FileCheck -# CHECK-NEXT: : 'RUN: at line 8'; echo 'foo bar' | FileCheck - -# CHECK: Command Output (stdout) -# CHECK: $ ":" "RUN: at line 1" -# CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line" -# CHECK-NEXT: $ ":" "RUN: at line 3" -# CHECK-NEXT: $ "echo" "foo bar" -# CHECK-NEXT: $ "FileCheck" "{{.*}}" -# CHECK-NEXT: $ ":" "RUN: at line 5" -# CHECK-NEXT: $ "echo" "foo baz" -# CHECK-NEXT: $ "FileCheck" "{{.*}}" -# CHECK-NOT: RUN +# TERSE: Command Output (stderr, truncated) +# TERSE: -- +# TERSE-NOT: RUN: at line 4 +# TERSE-NOT: true +# TERSE: RUN: at line 5 +# TERSE: false +# TERSE-NOT: RUN +# TERSE: NOTE: The failure may depend on previous RUN lines. + +# TERSE-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt + +# TERSE-NOT: Script: + +# TERSE: Command Output (stderr, truncated) +# TERSE-NOT: RUN: at line 4 +# TERSE: RUN: at line 6 +# TERSE: echo 'foo baz' +# TERSE: FileCheck +# TERSE-NOT: RUN +# TERSE: NOTE: The failure may depend on previous RUN lines. + +# TERSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt + +# TERSE-NOT: Script: + +# TERSE: Command Output (stdout, truncated) +# TERSE-NOT: $ ":" "RUN: at line 1" +# TERSE: $ ":" "RUN: at line 2" +# TERSE-NEXT: $ "false" +# TERSE-NOT: RUN +# TERSE: NOTE: The failure may depend on previous RUN lines. + +# TERSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt + +# TERSE-NOT: Script: + +# TERSE: Command Output (stdout, truncated) +# TERSE-NOT: $ ":" "RUN: at line 1" +# TERSE-NOT: $ ":" "RUN: at line 3" +# TERSE: $ ":" "RUN: at line 5" +# TERSE-NEXT: $ "echo" "foo baz" +# TERSE-NEXT: $ "FileCheck" "{{.*}}" +# TERSE-NOT: RUN +# TERSE: (NOTE: The failure may depend on previous RUN lines. +# TERSE-NEXT: Use --verbose to see previous RUN lines and outputs.) + +################################################################################ +# Checking lines for verbose output + +# VERBOSE: Testing: 4 tests + +# VERBOSE-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt + +# VERBOSE: Script: +# VERBOSE: RUN: at line 4{{.*}} true +# VERBOSE-NEXT: RUN: at line 5{{.*}} false +# VERBOSE-NEXT: RUN: at line 6{{.*}} true + +# VERBOSE: RUN: at line 4 +# VERBOSE: RUN: at line 5 +# VERBOSE-NOT: RUN: at line 6 +# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines. + +# VERBOSE-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt + +# VERBOSE: Script: +# VERBOSE: RUN: at line 4{{.*}} echo 'foo bar' | FileCheck +# VERBOSE-NEXT: RUN: at line 6{{.*}} echo 'foo baz' | FileCheck +# VERBOSE-NEXT: RUN: at line 9{{.*}} echo 'foo bar' | FileCheck + +# VERBOSE: RUN: at line 4 +# VERBOSE: RUN: at line 6 +# VERBOSE-NOT: RUN +# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines. + +# VERBOSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt + +# VERBOSE: Script: +# VERBOSE: : 'RUN: at line 1'; true +# VERBOSE-NEXT: : 'RUN: at line 2'; false +# VERBOSE-NEXT: : 'RUN: at line 3'; true + +# VERBOSE: Command Output (stdout) +# VERBOSE: $ ":" "RUN: at line 1" +# VERBOSE-NEXT: $ "true" +# VERBOSE-NEXT: $ ":" "RUN: at line 2" +# VERBOSE-NEXT: $ "false" +# VERBOSE-NOT: RUN +# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines. + +# VERBOSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt + +# VERBOSE: Script: +# VERBOSE: : 'RUN: at line 1'; : first line continued to second line +# VERBOSE-NEXT: : 'RUN: at line 3'; echo 'foo bar' | FileCheck +# VERBOSE-NEXT: : 'RUN: at line 5'; echo 'foo baz' | FileCheck +# VERBOSE-NEXT: : 'RUN: at line 8'; echo 'foo bar' | FileCheck + +# VERBOSE: Command Output (stdout) +# VERBOSE: $ ":" "RUN: at line 1" +# VERBOSE-NEXT: $ ":" "first" "line" "continued" "to" "second" "line" +# VERBOSE-NEXT: $ ":" "RUN: at line 3" +# VERBOSE-NEXT: $ "echo" "foo bar" +# VERBOSE-NEXT: $ "FileCheck" "{{.*}}" +# VERBOSE-NEXT: $ ":" "RUN: at line 5" +# VERBOSE-NEXT: $ "echo" "foo baz" +# VERBOSE-NEXT: $ "FileCheck" "{{.*}}" +# VERBOSE-NOT: RUN +# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines. diff --git a/llvm/utils/lit/tests/unittest-failing-locator.py b/llvm/utils/lit/tests/unittest-failing-locator.py new file mode 100644 --- /dev/null +++ b/llvm/utils/lit/tests/unittest-failing-locator.py @@ -0,0 +1,122 @@ +# Check that the locate_last_failing_run_line function works as expected. + +# RUN: %{python} %s +# END. + +import unittest + +from lit.TestRunner import locate_last_run_line + +class TestRunLineLocatorHeuristics(unittest.TestCase): + + def test_basic(self): + # from a script like + # RUN: echo Hello 1>&2 + basic = ( + "+ : 'RUN: at line 1'\n" + "+ echo Hello 1>&2\n" + "+ Hello\n" + ) + line_start, substr = locate_last_run_line(basic) + self.assertEqual(line_start, 0) + self.assertEqual(substr, "RUN: at line 1") + + def test_multiple(self): + # from a script like + # RUN: echo Hello 1>&2 + # RUN: false + multiple = ( + "+ : 'RUN: at line 1'\n" + "+ echo Hello 1>&2\n" + "+ Hello\n" + "+ : 'RUN: at line 2'\n" + "+ false\n" + ) + line_start, substr = locate_last_run_line(multiple) + self.assertEqual(line_start, multiple.rfind("+ :")) + self.assertEqual(substr, "RUN: at line 2") + + def test_varying_prefix(self): + # from a script like + # RUN: echo Hello 1>&2 + # RUN: false + # + # in a hypothetical shell which prints line-numbers on 'set +x' + # (as an example of something that varies) + varying_prefix = ( + "+ 1 : 'RUN: at line 1'\n" + "+ 2 echo Hello 1>&2\n" + "+ Hello\n" + "+ 3 : 'RUN: at line 2'\n" + "+ 4 false\n" + ) + line_start, substr = locate_last_run_line(varying_prefix) + self.assertEqual(line_start, varying_prefix.rfind("+ 3")) + self.assertEqual(substr, "RUN: at line 2") + + def test_confusing_basic(self): + # from a script like + # RUN: echo 'RUN: at line 10' 1>&2 + confusing_basic = ( + "+ : 'RUN: at line 1'\n" + "+ echo 'RUN: at line 10'\n" + "RUN: at line 10\n" + ) + line_start, substr = locate_last_run_line(confusing_basic) + # FIXME: These should both be equal ideally. + self.assertNotEqual(line_start, 0) + self.assertNotEqual(substr, "RUN: at line 1") + + def test_confusing_multiple(self): + # from a script like + # RUN: echo 'RUN: at line 10' 1>&2 + # RUN: false + confusing_multiple = ( + "+ : 'RUN: at line 1'\n" + "+ echo 'RUN: at line 10'\n" + "RUN: at line 10\n" + "+ : 'RUN: at line 2'\n" + "+ false\n" + ) + line_start, substr = locate_last_run_line(confusing_multiple) + self.assertEqual(line_start, confusing_multiple.rfind("+ :")) + self.assertEqual(substr, "RUN: at line 2") + + def test_confusing_varying_prefix_1(self): + # from a script like + # RUN: echo 'RUN: at line 10' 1>&2 + # RUN: false + # + # in a hypothetical shell which prints line-numbers on 'set +x' + # (as an example of something that varies) + confusing_varying_prefix = ( + "+ 1 : 'RUN: at line 1'\n" + "+ 2 echo 'RUN: at line 10'\n" + "RUN: at line 10\n" + "+ 3 : 'RUN: at line 2'\n" + "+ 4 false\n" + ) + line_start, substr = locate_last_run_line(confusing_varying_prefix) + self.assertEqual(line_start, confusing_varying_prefix.rfind("+ 3")) + self.assertEqual(substr, "RUN: at line 2") + + def test_confusing_varying_prefix_2(self): + # from a script like + # RUN: true + # RUN: not echo 'RUN: at line 100' + # + # in a hypothetical shell which prints line-numbers on 'set +x' + # (as an example of something that varies) + confusing_varying_prefix = ( + "+ 1 : 'RUN: at line 1'\n" + "+ 2 true\n" + "+ 3 : 'RUN: at line 2'\n" + "+ 4 not echo 'RUN: at line 100'\n" + "+ RUN: at line 100\n" + ) + line_start, substr = locate_last_run_line(confusing_varying_prefix) + # FIXME: These should both be equal ideally. + self.assertNotEqual(line_start, confusing_varying_prefix.rfind("+ 3")) + self.assertNotEqual(substr, "RUN: at line 2") + +unittest.main()