[lit] Extend internal diff to support -U

[jdenny]

Summary

When using lit's internal shell, RUN lines like the following
accidentally execute an external diff instead of lit's internal
diff:

# RUN: program | diff -U1 file -

Such cases exist now, in clang/test/Analysis for example. We are
preparing patches to ensure lit's internal diff is called in such
cases, which will then fail because lit's internal diff doesn't
recognize -U as a command-line option. This patch adds -U
support.

[rnk]

lgtm

[jdenny]

Rebased onto master so it doesn't depend on D66574 anymore. Thus, this now alters the original diff implementation within TestRunner.py instead of the diff.py that D66574 creates. This way, I don't have to apply the entire patch series at once, so handling bot failures should be easier.

It's basically the same patch, so another review shouldn't be necessary. I'll probably push later today.

llvm/utils/lit/lit/builtin_commands/diff.py

 import difflib
 import functools
 import getopt
 import locale
 import os
 import sys
 
 class DiffFlags():
     def __init__(self):
         self.ignore_all_space = False
         self.ignore_space_change = False
         self.unified_diff = False
+        self.num_context_lines = 3
         self.recursive_diff = False
         self.strip_trailing_cr = False
 
 def getDirTree(path, basedir=""):
     # Tree is a tuple of form (dirname, child_trees).
     # An empty dir has child_trees = [], a file has child_trees = None.
     child_trees = []
     for dirname, child_dirs, files in os.walk(os.path.join(basedir, path)):
except UnicodeDecodeError:
         except:
             return compareTwoBinaryFiles(flags, filepaths, filelines)
 
 def compareTwoBinaryFiles(flags, filepaths, filelines):
     #sys.stderr.write("Trying as binary....\n")
     exitCode = 0
     if hasattr(difflib, 'diff_bytes'):
         # python 3.5 or newer
-        diffs = difflib.diff_bytes(difflib.unified_diff, filelines[0], filelines[1], filepaths[0].encode(), filepaths[1].encode())
+        diffs = difflib.diff_bytes(difflib.unified_diff, filelines[0],
+                                   filelines[1], filepaths[0].encode(),
+                                   filepaths[1].encode(),
+                                   n = flags.num_context_lines)
         diffs = [diff.decode(errors="replace") for diff in diffs]
     else:
         # python 2.7
         if flags.unified_diff:
             func = difflib.unified_diff
         else:
             func = difflib.context_diff
-        diffs = func(filelines[0], filelines[1], filepaths[0], filepaths[1])
+        diffs = func(filelines[0], filelines[1], filepaths[0], filepaths[1],
+                     n = flags.num_context_lines)
 
     for diff in diffs:
         sys.stdout.write(diff)
         exitCode = 1
     return exitCode
 
 def compareTwoTextFiles(flags, filepaths, filelines_bin, encoding):
     #sys.stderr.write("Trying with encoding {}....\n".format(encoding))
if flags.ignore_all_space or flags.ignore_space_change:
         ignoreSpace = lambda line, separator: separator.join(line.split())
         ignoreAllSpaceOrSpaceChange = functools.partial(ignoreSpace, separator='' if flags.ignore_all_space else ' ')
         f = compose2(ignoreAllSpaceOrSpaceChange, f)
 
     for idx, lines in enumerate(filelines):
         filelines[idx]= [f(line) for line in lines]
 
     func = difflib.unified_diff if flags.unified_diff else difflib.context_diff
-    for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1]):
+    for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1],
+                     n = flags.num_context_lines):
         sys.stdout.write(diff)
         exitCode = 1
     return exitCode
 
 def printDirVsFile(dir_path, file_path):
     if os.path.getsize(file_path):
         msg = "File %s is a directory while file %s is a regular file"
     else:
while r < len(right_names):
         exitCode = 1
         printOnlyIn(right_base, right_tree[0], right_names[r])
         r += 1
     return exitCode
 
 def main(argv):
     args = argv[1:]
     try:
-        opts, args = getopt.gnu_getopt(args, "wbur", ["strip-trailing-cr"])
+        opts, args = getopt.gnu_getopt(args, "wbuU:r", ["strip-trailing-cr"])
     except getopt.GetoptError as err:
         sys.stderr.write("Unsupported: 'diff': %s\n" % str(err))
         sys.exit(1)
 
     flags = DiffFlags()
     filelines, filepaths, dir_trees = ([] for i in range(3))
     for o, a in opts:
         if o == "-w":
             flags.ignore_all_space = True
         elif o == "-b":
             flags.ignore_space_change = True
         elif o == "-u":
             flags.unified_diff = True
+        elif o.startswith("-U"):
+            flags.unified_diff = True
+            try:
+                flags.num_context_lines = int(a)
+                if flags.num_context_lines < 0:
+                    raise ValueException
+            except:
+                sys.stderr.write("Error: invalid '-U' argument: {}\n"
+                                 .format(a))
+                sys.exit(1)
         elif o == "-r":
             flags.recursive_diff = True
         elif o == "--strip-trailing-cr":
             flags.strip_trailing_cr = True
         else:
             assert False, "unhandled option"
 
     if len(args) != 2:

llvm/utils/lit/tests/Inputs/shtest-shell/diff-unified.txt

+# RUN: echo 1 > %t.foo
+# RUN: echo 2 >> %t.foo
+# RUN: echo 3 >> %t.foo
+# RUN: echo 4 >> %t.foo
+# RUN: echo 5 >> %t.foo
+# RUN: echo 6 foo >> %t.foo
+# RUN: echo 7 >> %t.foo
+# RUN: echo 8 >> %t.foo
+# RUN: echo 9 >> %t.foo
+# RUN: echo 10 >> %t.foo
+# RUN: echo 11 >> %t.foo
+
+# RUN: echo 1 > %t.bar
+# RUN: echo 2 >> %t.bar
+# RUN: echo 3 >> %t.bar
+# RUN: echo 4 >> %t.bar
+# RUN: echo 5 >> %t.bar
+# RUN: echo 6 bar >> %t.bar
+# RUN: echo 7 >> %t.bar
+# RUN: echo 8 >> %t.bar
+# RUN: echo 9 >> %t.bar
+# RUN: echo 10 >> %t.bar
+# RUN: echo 11 >> %t.bar
+
+# Default is 3 lines of context.
+# RUN: diff -u %t.foo %t.bar && false || true
+
+# Override default of 3 lines of context.
+# RUN: diff -U 2 %t.foo %t.bar && false || true
+# RUN: diff -U4 %t.foo %t.bar && false || true
+# RUN: diff -U0 %t.foo %t.bar && false || true
+
+# Check bad -U argument.
+# RUN: diff -U 30.1 %t.foo %t.foo && false || true
+# RUN: diff -U-1 %t.foo %t.foo && false || true
+
+# Fail so lit will print output.
+# RUN: false

llvm/utils/lit/tests/max-failures.py

 # Check the behavior of --max-failures option.
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/max-failures > %t.out
 # RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/max-failures 2>> %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
-# CHECK: Failing Tests (30)
+# CHECK: Failing Tests (31)
 # CHECK: Failing Tests (1)
 # CHECK: Failing Tests (2)
 # CHECK: error: Option '--max-failures' requires positive integer

llvm/utils/lit/tests/shtest-shell.py

 # CHECK: *** TEST 'shtest-shell :: diff-r-error-8.txt' FAILED ***
 # CHECK: $ "diff" "-r" "{{[^"]*}}" "-"
 # CHECK: # command stderr:
 # CHECK: Error: cannot recursively compare '-'
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: PASS: shtest-shell :: diff-r.txt
 
+
+# CHECK: FAIL: shtest-shell :: diff-unified.txt
+
+# CHECK: *** TEST 'shtest-shell :: diff-unified.txt' FAILED ***
+
+# CHECK: $ "diff" "-u" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 3
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-NEXT: 9
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U" "2" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U4" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 2
+# CHECK-NEXT: 3
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-NEXT: 9
+# CHECK-NEXT: 10
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U0" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U" "30.1" "{{[^"]*}}" "{{[^"]*}}"
+# CHECK: # command stderr:
+# CHECK: Error: invalid '-U' argument: 30.1
+# CHECK: error: command failed with exit status: 1
+# CHECK: $ "true"
+
+# CHECK: $ "diff" "-U-1" "{{[^"]*}}" "{{[^"]*}}"
+# CHECK: # command stderr:
+# CHECK: Error: invalid '-U' argument: -1
+# CHECK: error: command failed with exit status: 1
+# CHECK: $ "true"
+
+# CHECK: $ "false"
+
+# CHECK: ***
+
+
 # CHECK: FAIL: shtest-shell :: error-0.txt
 # CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
 # CHECK: $ "not-a-real-command"
 # CHECK: # command stderr:
 # CHECK: 'not-a-real-command': command not found
 # CHECK: error: command failed with exit status: 127
 # CHECK: ***
 

 # CHECK: *** TEST 'shtest-shell :: rm-error-3.txt' FAILED ***
 # CHECK: Exit Code: 1
 # CHECK: ***
 
 # CHECK: PASS: shtest-shell :: rm-unicode-0.txt
 # CHECK: PASS: shtest-shell :: sequencing-0.txt
 # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
 # CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failing Tests (30)
+# CHECK: Failing Tests (31)