[lit] Extend internal diff to support -U

[jdenny]

Summary

When using lit's internal shell, RUN lines like the following
accidentally execute an external diff instead of lit's internal
diff:

# RUN: program | diff -U1 file -

Such cases exist now, in clang/test/Analysis for example. We are
preparing patches to ensure lit's internal diff is called in such
cases, which will then fail because lit's internal diff doesn't
recognize -U as a command-line option. This patch adds -U
support.

[rnk]

lgtm

[jdenny]

Rebased onto master so it doesn't depend on D66574 anymore. Thus, this now alters the original diff implementation within TestRunner.py instead of the diff.py that D66574 creates. This way, I don't have to apply the entire patch series at once, so handling bot failures should be easier.

It's basically the same patch, so another review shouldn't be necessary. I'll probably push later today.

llvm/utils/lit/lit/TestRunner.py

for dir in args:
                 stderr.write("Error: 'mkdir' command failed, %s\n" % str(err))
                 exitCode = 1
     return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False)
 
 def executeBuiltinDiff(cmd, cmd_shenv):
     """executeBuiltinDiff - Compare files line by line."""
     args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:]
     try:
-        opts, args = getopt.gnu_getopt(args, "wbur", ["strip-trailing-cr"])
+        opts, args = getopt.gnu_getopt(args, "wbuU:r", ["strip-trailing-cr"])
     except getopt.GetoptError as err:
         raise InternalShellError(cmd, "Unsupported: 'diff':  %s" % str(err))
 
     filelines, filepaths, dir_trees = ([] for i in range(3))
     ignore_all_space = False
     ignore_space_change = False
     unified_diff = False
+    num_context_lines = 3
     recursive_diff = False
     strip_trailing_cr = False
     for o, a in opts:
         if o == "-w":
             ignore_all_space = True
         elif o == "-b":
             ignore_space_change = True
         elif o == "-u":
             unified_diff = True
+        elif o.startswith("-U"):
+            unified_diff = True
+            try:
+                num_context_lines = int(a)
+                if num_context_lines < 0:
+                    raise ValueException
+            except:
+                raise InternalShellError(cmd,
+                                         "Error: invalid '-U' argument: {}\n"
+                                         .format(a))
         elif o == "-r":
             recursive_diff = True
         elif o == "--strip-trailing-cr":
             strip_trailing_cr = True
         else:
             assert False, "unhandled option"
 
     if len(args) != 2:
def compareTwoBinaryFiles(filepaths):
         filelines = []
         for file in filepaths:
             with open(file, 'rb') as f:
                 filelines.append(f.readlines())
 
         exitCode = 0
         if hasattr(difflib, 'diff_bytes'):
             # python 3.5 or newer
-            diffs = difflib.diff_bytes(difflib.unified_diff, filelines[0], filelines[1], filepaths[0].encode(), filepaths[1].encode())
+            diffs = difflib.diff_bytes(difflib.unified_diff, filelines[0],
+                                       filelines[1], filepaths[0].encode(),
+                                       filepaths[1].encode(),
+                                       n = num_context_lines)
             diffs = [diff.decode() for diff in diffs]
         else:
             # python 2.7
             func = difflib.unified_diff if unified_diff else difflib.context_diff
-            diffs = func(filelines[0], filelines[1], filepaths[0], filepaths[1])
+            diffs = func(filelines[0], filelines[1], filepaths[0], filepaths[1],
+                         n = num_context_lines)
 
         for diff in diffs:
             stdout.write(diff)
             exitCode = 1
         return exitCode
 
     def compareTwoTextFiles(filepaths, encoding):
         filelines = []
def compareTwoTextFiles(filepaths, encoding):
             ignoreSpace = lambda line, separator: separator.join(line.split())
             ignoreAllSpaceOrSpaceChange = functools.partial(ignoreSpace, separator='' if ignore_all_space else ' ')
             f = compose2(ignoreAllSpaceOrSpaceChange, f)
 
         for idx, lines in enumerate(filelines):
             filelines[idx]= [f(line) for line in lines]
 
         func = difflib.unified_diff if unified_diff else difflib.context_diff
-        for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1]):
+        for diff in func(filelines[0], filelines[1], filepaths[0], filepaths[1],
+                         n = num_context_lines):
             stdout.write(diff)
             exitCode = 1
         return exitCode
 
     def printDirVsFile(dir_path, file_path):
         if os.path.getsize(file_path):
             msg = "File %s is a directory while file %s is a regular file"
         else:

llvm/utils/lit/tests/Inputs/shtest-shell/diff-unified-error-0.txt

+# Check bad -U argument.
+# RUN: echo foo > %t
+# RUN: diff -U 30.1 %t %t

llvm/utils/lit/tests/Inputs/shtest-shell/diff-unified-error-1.txt

+# Check bad -U argument.
+# RUN: echo foo > %t
+# RUN: diff -U-1 %t %t

llvm/utils/lit/tests/Inputs/shtest-shell/diff-unified.txt

+# RUN: echo 1 > %t.foo
+# RUN: echo 2 >> %t.foo
+# RUN: echo 3 >> %t.foo
+# RUN: echo 4 >> %t.foo
+# RUN: echo 5 >> %t.foo
+# RUN: echo 6 foo >> %t.foo
+# RUN: echo 7 >> %t.foo
+# RUN: echo 8 >> %t.foo
+# RUN: echo 9 >> %t.foo
+# RUN: echo 10 >> %t.foo
+# RUN: echo 11 >> %t.foo
+
+# RUN: echo 1 > %t.bar
+# RUN: echo 2 >> %t.bar
+# RUN: echo 3 >> %t.bar
+# RUN: echo 4 >> %t.bar
+# RUN: echo 5 >> %t.bar
+# RUN: echo 6 bar >> %t.bar
+# RUN: echo 7 >> %t.bar
+# RUN: echo 8 >> %t.bar
+# RUN: echo 9 >> %t.bar
+# RUN: echo 10 >> %t.bar
+# RUN: echo 11 >> %t.bar
+
+# Default is 3 lines of context.
+# RUN: diff -u %t.foo %t.bar && false || true
+
+# Override default of 3 lines of context.
+# RUN: diff -U 2 %t.foo %t.bar && false || true
+# RUN: diff -U4 %t.foo %t.bar && false || true
+# RUN: diff -U0 %t.foo %t.bar && false || true
+
+# Fail so lit will print output.
+# RUN: false

llvm/utils/lit/tests/max-failures.py

 # Check the behavior of --max-failures option.
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/max-failures > %t.out
 # RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/max-failures >> %t.out
 # RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/max-failures 2>> %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
-# CHECK: Failing Tests (27)
+# CHECK: Failing Tests (30)
 # CHECK: Failing Tests (1)
 # CHECK: Failing Tests (2)
 # CHECK: error: argument --max-failures: requires positive integer, but found '0'

llvm/utils/lit/tests/shtest-shell.py

 # CHECK: *** TEST 'shtest-shell :: diff-r-error-6.txt' FAILED ***
 # CHECK: $ "diff" "-r" 
 # CHECK: # command output:
 # CHECK: File {{.*}}dir1{{.*}}extra_file is a regular empty file while file {{.*}}dir2{{.*}}extra_file is a directory
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: PASS: shtest-shell :: diff-r.txt
 
+# CHECK: FAIL: shtest-shell :: diff-unified-error-0.txt
+# CHECK: *** TEST 'shtest-shell :: diff-unified-error-0.txt' FAILED ***
+# CHECK: $ "diff" "-U" "30.1" "{{[^"]*}}" "{{[^"]*}}"
+# CHECK: # command stderr:
+# CHECK: Error: invalid '-U' argument: 30.1
+# CHECK: error: command failed with exit status: 127
+# CHECK: ***
+
+# CHECK: FAIL: shtest-shell :: diff-unified-error-1.txt
+# CHECK: *** TEST 'shtest-shell :: diff-unified-error-1.txt' FAILED ***
+# CHECK: $ "diff" "-U-1" "{{[^"]*}}" "{{[^"]*}}"
+# CHECK: # command stderr:
+# CHECK: Error: invalid '-U' argument: -1
+# CHECK: error: command failed with exit status: 127
+# CHECK: ***
+
+
+# CHECK: FAIL: shtest-shell :: diff-unified.txt
+
+# CHECK: *** TEST 'shtest-shell :: diff-unified.txt' FAILED ***
+
+# CHECK: $ "diff" "-u" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 3
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-NEXT: 9
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U" "2" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U4" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: 2
+# CHECK-NEXT: 3
+# CHECK-NEXT: 4
+# CHECK-NEXT: 5
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-NEXT: 7
+# CHECK-NEXT: 8
+# CHECK-NEXT: 9
+# CHECK-NEXT: 10
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "diff" "-U0" "{{[^"]*}}.foo" "{{[^"]*}}.bar"
+# CHECK: # command output:
+# CHECK: @@ {{.*}} @@
+# CHECK-NEXT: -6 foo
+# CHECK-NEXT: +6 bar
+# CHECK-EMPTY:
+# CHECK-NEXT: error: command failed with exit status: 1
+# CHECK-NEXT: $ "true"
+
+# CHECK: $ "false"
+
+# CHECK: ***
+
+
 # CHECK: FAIL: shtest-shell :: error-0.txt
 # CHECK: *** TEST 'shtest-shell :: error-0.txt' FAILED ***
 # CHECK: $ "not-a-real-command"
 # CHECK: # command stderr:
 # CHECK: 'not-a-real-command': command not found
 # CHECK: error: command failed with exit status: 127
 # CHECK: ***
 

 # CHECK: *** TEST 'shtest-shell :: rm-error-3.txt' FAILED ***
 # CHECK: Exit Code: 1
 # CHECK: ***
 
 # CHECK: PASS: shtest-shell :: rm-unicode-0.txt
 # CHECK: PASS: shtest-shell :: sequencing-0.txt
 # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
 # CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failing Tests (27)
+# CHECK: Failing Tests (30)