[lit] Fix `not` calling internal commands

[jdenny]

Summary

Without this patch, when using lit's internal shell, if not on a lit
RUN line calls env, diff, or any of the other in-process shell
builtins that lit implements, lit accidentally searches for the latter
as an external executable. What's worse is that works fine when a
developer is testing on a platform where those executables are
available and behave as expected, but it then breaks on other
platforms.

not seems useful for some builtins, such as diff, so this patch
supports such uses. not --crash does not seem useful for builtins,
so this patch diagnoses such uses. In all cases, this patch ensures
shell builtins are found behind any sequence of env and not
commands.

not calling env calling an external command appears useful when
the env and external command are part of a lit substitution, as in
D65156. This patch supports that by looking through any sequence of
env and not commands, building the environment from the envs,
and storing the nots. The nots are then added back to the command
line without the envs to execute externally. This avoids the need
to replicate the not implementation, in particular the --crash
option, in lit.

[rnk]

lgtm

[rnk]

I managed to run into this again:
https://reviews.llvm.org/rL371478#687359

Is anything blocking this?

[jdenny]

In D66531#1665158, @rnk wrote:

I managed to run into this again:
https://reviews.llvm.org/rL371478#687359

Is anything blocking this?

See D65697#1640136. The next step is that D66574 needs to be reviewed, and I have some more patches I need to propose after that to add some missing features to the internal diff implementation. I've been very busy, so I haven't been pinging, but I didn't realize others were affected.

[jdenny]

• Rebased onto recent master.
• Adjusted tests not to use diff as an example in-process builtin because it's now an out-of-process builtin.
• Fixed discovery of out-of-process builtins behind not.

llvm/utils/lit/lit/TestRunner.py

 import threading
 
 import io
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
-from lit.ShCommands import GlobItem
+from lit.ShCommands import GlobItem, Command
 import lit.ShUtil as ShUtil
 import lit.Test as Test
 import lit.util
 from lit.util import to_bytes, to_string, to_unicode
 from lit.BooleanExpression import BooleanExpression
 
 class InternalShellError(Exception):
     def __init__(self, command, message):
inproc_builtins = {'cd': executeBuiltinCd,
                        ':': executeBuiltinColon}
     # To avoid deadlock, we use a single stderr stream for piped
     # output. This is null until we have seen some output using
     # stderr.
     for i,j in enumerate(cmd.commands):
         # Reference the global environment by default.
         cmd_shenv = shenv
         args = list(j.args)
-        while args[0] == 'env':
-            # Create a copy of the global environment and modify it for this one
-            # command. There might be multiple envs in a pipeline, and
-            # there might be multiple envs in a command (usually when one comes
-            # from a substitution):
+        not_args = []
+        not_count = 0
+        not_crash = False
+        while True:
+            if args[0] == 'env':
+                # Create a copy of the global environment and modify it for
+                # this one command. There might be multiple envs in a pipeline,
+                # and there might be multiple envs in a command (usually when
+                # one comes from a substitution):
                 #   env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
                 #   env FOO=1 %{another_env_plus_cmd} | FileCheck %s
                 if cmd_shenv is shenv:
                     cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
                 args = updateEnv(cmd_shenv, args)
                 if not args:
-                raise InternalShellError(j,
-                                         "Error: 'env' requires a subcommand")
+                    raise InternalShellError(j, "Error: 'env' requires a"
+                                                " subcommand")
+            elif args[0] == 'not':
+                not_args.append(args.pop(0))
+                not_count += 1
+                if args and args[0] == '--crash':
+                    not_args.append(args.pop(0))
+                    not_crash = True
+                if not args:
+                    raise InternalShellError(j, "Error: 'not' requires a"
+                                                " subcommand")
+            else:
+                break
 
         # Handle in-process builtins.
         #
         # Handle "echo" as a builtin if it is not part of a pipeline. This
         # greatly speeds up tests that construct input files by repeatedly
         # echo-appending to a file.
         # FIXME: Standardize on the builtin echo implementation. We can use a
         # temporary file to sidestep blocking pipe write issues.
         inproc_builtin = inproc_builtins.get(args[0], None)
         if inproc_builtin and (args[0] != 'echo' or len(cmd.commands) == 1):
             # env calling an in-process builtin is useless, so we take the safe
             # approach of complaining.
             if not cmd_shenv is shenv:
                 raise InternalShellError(j, "Error: 'env' cannot call '{}'"
                                             .format(args[0]))
+            if not_crash:
+                raise InternalShellError(j, "Error: 'not --crash' cannot call"
+                                            " '{}'".format(args[0]))
             if len(cmd.commands) != 1:
                 raise InternalShellError(j, "Unsupported: '{}' cannot be part"
                                             " of a pipeline".format(args[0]))
-            result = inproc_builtin(j, cmd_shenv)
+            result = inproc_builtin(Command(args, j.redirects), cmd_shenv)
+            if not_count % 2:
+                result.exitCode = int(not result.exitCode)
+            result.command.args = j.args;
             results.append(result)
             return result.exitCode
 
+        # Resolve any out-of-process builtin command before adding back 'not'
+        # commands.
+        if args[0] in builtin_commands:
+            args.insert(0, sys.executable)
+            cmd_shenv.env['PYTHONPATH'] = \
+                os.path.dirname(os.path.abspath(__file__))
+            args[1] = os.path.join(builtin_commands_dir, args[1] + ".py")
+
+        # We had to search through the 'not' commands to find all the 'env'
+        # commands and any other in-process builtin command.  We don't want to
+        # reimplement 'not' and its '--crash' here, so just push all 'not'
+        # commands back to be called as external commands.  Because this
+        # approach effectively moves all 'env' commands up front, it relies on
+        # the assumptions that (1) environment variables are not intended to be
+        # relevant to 'not' commands and (2) the 'env' command should always
+        # blindly pass along the status it receives from any command it calls.
+        args = not_args + args
+
         stdin, stdout, stderr = processRedirects(j, default_stdin, cmd_shenv,
                                                  opened_files)
 
         # If stderr wants to come from stdout, but stdout isn't a pipe, then put
         # stderr on a pipe and treat it as stdout.
         if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
             stderr = subprocess.PIPE
             stderrIsStdout = True
         else:
             stderrIsStdout = False
 
             # Don't allow stderr on a PIPE except for the last
             # process, this could deadlock.
             #
             # FIXME: This is slow, but so is deadlock.
             if stderr == subprocess.PIPE and j != cmd.commands[-1]:
                 stderr = tempfile.TemporaryFile(mode='w+b')
                 stderrTempFiles.append((i, stderr))
 
         # Resolve the executable path ourselves.
         executable = None
-        is_builtin_cmd = args[0] in builtin_commands;
-        if not is_builtin_cmd:
-            # For paths relative to cwd, use the cwd of the shell environment.
-            if args[0].startswith('.'):
-                exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0])
-                if os.path.isfile(exe_in_cwd):
-                    executable = exe_in_cwd
-            if not executable:
-                executable = lit.util.which(args[0], cmd_shenv.env['PATH'])
-            if not executable:
-                raise InternalShellError(j, '%r: command not found' % args[0])
 
         # Replace uses of /dev/null with temporary files.
         if kAvoidDevNull:
             # In Python 2.x, basestring is the base class for all string (including unicode)
             # In Python 3.x, basestring no longer exist and str is always unicode
             try:
                 str_type = basestring
             except NameError:
                 str_type = str
             for i,arg in enumerate(args):
                 if isinstance(arg, str_type) and kDevNull in arg:
                     f = tempfile.NamedTemporaryFile(delete=False)
                     f.close()
                     named_temp_files.append(f.name)
                     args[i] = arg.replace(kDevNull, f.name)
 
         # Expand all glob expressions
         args = expand_glob_expressions(args, cmd_shenv.cwd)
-        if is_builtin_cmd:
-            args.insert(0, sys.executable)
-            cmd_shenv.env['PYTHONPATH'] = \
-                os.path.dirname(os.path.abspath(__file__))
-            args[1] = os.path.join(builtin_commands_dir ,args[1] + ".py")
 
         # On Windows, do our own command line quoting for better compatibility
         # with some core utility distributions.
         if kIsWindows:
             args = quote_windows_command(args)
 
         try:
             procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd,

llvm/utils/lit/tests/Inputs/shtest-env/env-calls-not-builtin.txt

+# Other tests thoroughly check that 'env' cannot call various builtin commands.
+# Pick one and make sure it fails even if there's a 'not' in the way.
+
+# RUN: env -u FOO BAR=3 not rm %t.no-such-file

llvm/utils/lit/tests/Inputs/shtest-not/fail.py

+#!/usr/bin/env python
+
+import print_environment
+import sys
+
+print_environment.execute()
+sys.exit(1)

llvm/utils/lit/tests/Inputs/shtest-not/lit.cfg

+import lit.formats
+config.name = 'shtest-not'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))

llvm/utils/lit/tests/Inputs/shtest-not/not-args-last-is-crash.txt

+# RUN: not --crash

llvm/utils/lit/tests/Inputs/shtest-not/not-args-nested-none.txt

+# RUN: not not not

llvm/utils/lit/tests/Inputs/shtest-not/not-args-none.txt

+# RUN: not

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-cd.txt

+# Internal cd always succeeds.
+# RUN: not not cd foobar
+# RUN: not --crash cd foobar

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-colon.txt

+# ":" always succeeds.
+# RUN: not not : foobar
+# RUN: not --crash :

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff-with-crash.txt

+# Lit's diff is out-of-process, so check that "not --crash diff" fails because
+# diff doesn't crash rather than because "not --crash diff" isn't supported.
+
+# RUN: echo 'foo' > %t.foo
+# RUN: echo 'bar' > %t.bar
+# RUN: not --crash diff -u %t.foo %t.bar

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff.txt

+# RUN: echo 'foo' > %t.foo0
+# RUN: echo 'foo' > %t.foo1
+# RUN: echo 'bar' > %t.bar
+
+# diff fails.
+# RUN: not diff %t.foo0 %t.bar
+# RUN: not not not diff %t.foo0 %t.bar
+# RUN: not not not not not diff %t.foo0 %t.bar
+
+# diff succeeds.
+# RUN: diff %t.foo0 %t.foo1
+# RUN: not not diff %t.foo0 %t.foo1
+# RUN: not not not not diff %t.foo0 %t.foo1
+
+# diff succeeds but we expect failure.
+# RUN: not diff %t.foo0 %t.foo1

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-echo.txt

+# Internal "echo" always succeeds.
+# RUN: not not echo hello world
+# RUN: not --crash echo hello world

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-env-builtin.txt

+# Other tests thoroughly check that 'not' cannot call various builtin commands.
+# Pick one and make sure it fails even if there's an 'env' in the way.
+
+# RUN: not --crash env -u FOO BAR=3 rm %t.no-such-file

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-export.txt

+# Internal "export" always succeeds.
+# RUN: not not export FOO=1
+# RUN: not --crash export BAZ=3

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-external.txt

+# RUN: echo foo > %t.in
+# RUN: echo bar >> %t.in
+
+
+# Simple uses of 'not'
+
+# RUN: not %{python} fail.py
+# RUN: not not %{python} pass.py
+# RUN: not not not %{python} fail.py
+# RUN: not not not not %{python} pass.py
+
+
+# Simple uses of 'not --crash'
+
+# RUN: not not --crash %{python} pass.py
+# RUN: not not --crash %{python} fail.py
+# RUN: not not --crash not %{python} pass.py
+# RUN: not not --crash not  %{python} fail.py
+
+
+# Various patterns of 'not' and 'env'
+#
+# There's no particular pattern to the 'env' arguments except we try not to
+# do the same thing every time.
+
+# RUN: env not %{python} fail.py | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+# RUN: not env %{python} fail.py | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+
+# RUN: env FOO=1 not %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR-NO %s
+
+# RUN: not env FOO=1 BAR=1 %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR1 %s
+
+# RUN: env FOO=1 BAR=1 not env -u FOO BAR=2 %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR2 %s
+
+# RUN: not env FOO=1 BAR=1 not env -u FOO -u BAR %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+
+# RUN: not not env FOO=1 env FOO=2 BAR=1 %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO2,BAR1 %s
+
+# RUN: env FOO=1 -u BAR env -u FOO BAR=1 not not %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR1 %s
+
+
+# Various patterns of 'not', 'not --crash', and 'env'
+
+# RUN: not env FOO=1 BAR=1 env FOO=2 BAR=2 not --crash %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO2,BAR2 %s
+
+# RUN: not env FOO=1 BAR=1 not --crash not %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR1 %s
+
+# RUN: not not --crash env -u BAR not env -u FOO BAR=1 %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR1 %s
+
+
+# FOO-NO: FOO = [undefined]
+# FOO1:   FOO = 1
+# FOO2:   FOO = 2
+
+# BAR-NO: BAR = [undefined]
+# BAR1:   BAR = 1
+# BAR2:   BAR = 2

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-mkdir.txt

+# RUN: not mkdir %t/foobar
+# RUN: not --crash mkdir foobar

llvm/utils/lit/tests/Inputs/shtest-not/not-calls-rm.txt

+# RUN: not rm %t
+# RUN: not --crash rm foobar

llvm/utils/lit/tests/Inputs/shtest-not/pass.py

+#!/usr/bin/env python
+
+import print_environment
+
+print_environment.execute()

llvm/utils/lit/tests/Inputs/shtest-not/print_environment.py

+from __future__ import print_function
+import os
+
+def execute():
+    for name in ['FOO', 'BAR']:
+        print(name, '=', os.environ.get(name, '[undefined]'))

llvm/utils/lit/tests/shtest-env.py

 # Check the env command
 #
 # RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-env \
 # RUN: | FileCheck -match-full-lines %s
 #
 # END.
 
 # Make sure env commands are included in printed commands.
 
-# CHECK: -- Testing: 15 tests{{.*}}
+# CHECK: -- Testing: 16 tests{{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-assign.txt ({{[^)]*}})
 # CHECK: $ "env" "FOO=1"
 # CHECK: Error: 'env' requires a subcommand
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-u-arg.txt ({{[^)]*}})
 # CHECK: $ "env" "-u" "FOO"

 # CHECK: Error: 'env' cannot call 'export'
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: FAIL: shtest-env :: env-calls-mkdir.txt ({{[^)]*}})
 # CHECK: $ "env" "-u" "FOO" "BAR=3" "mkdir" "foobar"
 # CHECK: Error: 'env' cannot call 'mkdir'
 # CHECK: error: command failed with exit status: {{.*}}
 
+# CHECK: FAIL: shtest-env :: env-calls-not-builtin.txt ({{[^)]*}})
+# CHECK: $ "env" "-u" "FOO" "BAR=3" "not" "rm" "{{.*}}.no-such-file"
+# CHECK: Error: 'env' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
 # CHECK: FAIL: shtest-env :: env-calls-rm.txt ({{[^)]*}})
 # CHECK: $ "env" "-u" "FOO" "BAR=3" "rm" "foobar"
 # CHECK: Error: 'env' cannot call 'rm'
 # CHECK: error: command failed with exit status: {{.*}}
 
 # CHECK: PASS: shtest-env :: env-u.txt ({{[^)]*}})
 # CHECK: $ "{{[^"]*}}" "print_environment.py"
 # CHECK: $ "env" "-u" "FOO" "{{[^"]*}}" "print_environment.py"
 # CHECK: $ "env" "-u" "FOO" "-u" "BAR" "{{[^"]*}}" "print_environment.py"
 # CHECK-NOT: ${{.*}}print_environment.py
 
 # CHECK: PASS: shtest-env :: env.txt ({{[^)]*}})
 # CHECK: $ "env" "A_FOO=999" "{{[^"]*}}" "print_environment.py"
 # CHECK: $ "env" "A_FOO=1" "B_BAR=2" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
 # CHECK-NOT: ${{.*}}print_environment.py
 
 # CHECK: PASS: shtest-env :: mixed.txt ({{[^)]*}})
 # CHECK: $ "env" "A_FOO=999" "-u" "FOO" "{{[^"]*}}" "print_environment.py"
 # CHECK: $ "env" "A_FOO=1" "-u" "FOO" "B_BAR=2" "-u" "BAR" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
 # CHECK-NOT: ${{.*}}print_environment.py
 
 # CHECK: Expected Passes : 4
-# CHECK: Unexpected Failures: 11
+# CHECK: Unexpected Failures: 12
 # CHECK-NOT: {{.}}

llvm/utils/lit/tests/shtest-not.py

+# Check the not command
+#
+# RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-not \
+# RUN: | FileCheck -match-full-lines %s
+#
+# END.
+
+# Make sure not and env commands are included in printed commands.
+
+# CHECK: -- Testing: 13 tests{{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-last-is-crash.txt {{.*}}
+# CHECK: $ "not" "--crash"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-nested-none.txt {{.*}}
+# CHECK: $ "not" "not" "not"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-none.txt {{.*}}
+# CHECK: $ "not"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-cd.txt {{.*}}
+# CHECK: $ "not" "not" "cd" "foobar"
+# CHECK: $ "not" "--crash" "cd" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'cd'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-colon.txt {{.*}}
+# CHECK: $ "not" "not" ":" "foobar"
+# CHECK: $ "not" "--crash" ":"
+# CHECK: Error: 'not --crash' cannot call ':'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-diff-with-crash.txt {{.*}}
+# CHECK: $ "not" "--crash" "diff" "-u" {{.*}}
+# CHECK-NOT: "$"
+# CHECK-NOT: {{[Ee]rror}}
+# CHECK: error: command failed with exit status: {{.*}}
+# CHECK-NOT: {{[Ee]rror}}
+# CHECK-NOT: "$"
+
+# CHECK: FAIL: shtest-not :: not-calls-diff.txt {{.*}}
+# CHECK: $ "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "diff" {{.*}}
+# CHECK: $ "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "diff" {{.*}}
+# CHECK-NOT: "$"
+
+# CHECK: FAIL: shtest-not :: not-calls-echo.txt {{.*}}
+# CHECK: $ "not" "not" "echo" "hello" "world"
+# CHECK: $ "not" "--crash" "echo" "hello" "world"
+# CHECK: Error: 'not --crash' cannot call 'echo'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-env-builtin.txt {{.*}}
+# CHECK: $ "not" "--crash" "env" "-u" "FOO" "BAR=3" "rm" "{{.*}}.no-such-file"
+# CHECK: Error: 'env' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-export.txt {{.*}}
+# CHECK: $ "not" "not" "export" "FOO=1"
+# CHECK: $ "not" "--crash" "export" "BAZ=3"
+# CHECK: Error: 'not --crash' cannot call 'export'
+# CHECK: error: command failed with exit status: {{.*}}
+
+
+# CHECK: PASS: shtest-not :: not-calls-external.txt {{.*}}
+
+# CHECK: $ "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "not" "not" "{{[^"]*}}" "pass.py"
+
+# CHECK: $ "not" "not" "--crash" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "--crash" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "not" "{{[^"]*}}" "fail.py"
+
+# CHECK: $ "env" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "{{[^"]*}}" "fail.py"
+# CHECK: $ "env" "FOO=1" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "{{[^"]*}}" "fail.py"
+# CHECK: $ "env" "FOO=1" "BAR=1" "not" "env" "-u" "FOO" "BAR=2" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "not" "env" "-u" "FOO" "-u" "BAR" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "env" "FOO=1" "env" "FOO=2" "BAR=1" "{{[^"]*}}" "pass.py"
+# CHECK: $ "env" "FOO=1" "-u" "BAR" "env" "-u" "FOO" "BAR=1" "not" "not" "{{[^"]*}}" "pass.py"
+
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "env" "FOO=2" "BAR=2" "not" "--crash" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "not" "--crash" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "env" "-u" "BAR" "not" "env" "-u" "FOO" "BAR=1" "{{[^"]*}}" "pass.py"
+
+
+# CHECK: FAIL: shtest-not :: not-calls-mkdir.txt {{.*}}
+# CHECK: $ "not" "mkdir" {{.*}}
+# CHECK: $ "not" "--crash" "mkdir" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'mkdir'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-rm.txt {{.*}}
+# CHECK: $ "not" "rm" {{.*}}
+# CHECK: $ "not" "--crash" "rm" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: Expected Passes : 1
+# CHECK: Unexpected Failures: 12
+# CHECK-NOT: {{.}}