[lit] Print internal env commands

[jdenny]

Summary

Without this patch, the internal env command removes env and its
args from the command line while parsing it. This patch modifies a
copy instead so that the original command line is printed.

[rnk]

Suggestion, but this looks good too.

llvm/utils/lit/lit/TestRunner.py

882

I think it would be clearer to make updateEnv return the new command to run after env instead of modifying cmd.args. We copy cmd.args on line 903 currently anyway.

915

If you take my suggestion, this needs to use args not j.args, since it will refer to "env" and not the real exe for env-prefixed commands.

[jdenny]

Apply rnk's suggestion, and adjust updateEnv to make it clear it only operates on the arguments.

@rnk, @stella.stamenova : thanks.

I'll wait a bit to push in case there are further comments.

[rnk]

lgtm, thanks!

llvm/utils/lit/lit/TestRunner.py

for arg in seq:
             result.extend(bs_buf)
 
         if needquote:
             result.extend(bs_buf)
             result.append('"')
 
     return ''.join(result)
 
-# cmd is export or env
-def updateEnv(env, cmd):
+# args are from 'export' or 'env' command.
+# Returns copy of args without those commands or their arguments.
+def updateEnv(env, args):
     arg_idx = 1
     unset_next_env_var = False
-    for arg_idx, arg in enumerate(cmd.args[1:]):
+    for arg_idx, arg in enumerate(args[1:]):
         # Support for the -u flag (unsetting) for env command
         # e.g., env -u FOO -u BAR will remove both FOO and BAR
         # from the environment.
         if arg == '-u':
             unset_next_env_var = True
             continue
         if unset_next_env_var:
             unset_next_env_var = False
             if arg in env.env:
                 del env.env[arg]
             continue
 
         # Partition the string into KEY=VALUE.
         key, eq, val = arg.partition('=')
         # Stop if there was no equals.
         if eq == '':
             break
         env.env[key] = val
-    cmd.args = cmd.args[arg_idx+1:]
+    return args[arg_idx+1:]
 
 def executeBuiltinEcho(cmd, shenv):
     """Interpret a redirected echo command"""
     opened_files = []
     stdin, stdout, stderr = processRedirects(cmd, subprocess.PIPE, shenv,
                                              opened_files)
     if stdin != subprocess.PIPE or stderr != subprocess.PIPE:
         raise InternalShellError(
if cmd.commands[0].args[0] == 'echo' and len(cmd.commands) == 1:
                                           False))
         return 0
 
     if cmd.commands[0].args[0] == 'export':
         if len(cmd.commands) != 1:
             raise ValueError("'export' cannot be part of a pipeline")
         if len(cmd.commands[0].args) != 2:
             raise ValueError("'export' supports only one argument")
-        updateEnv(shenv, cmd.commands[0])
+        updateEnv(shenv, cmd.commands[0].args)
         return 0
 
     if cmd.commands[0].args[0] == 'mkdir':
         if len(cmd.commands) != 1:
             raise InternalShellError(cmd.commands[0], "Unsupported: 'mkdir' "
                                      "cannot be part of a pipeline")
         cmdResult = executeBuiltinMkdir(cmd.commands[0], shenv)
         results.append(cmdResult)
def _executeShCmd(cmd, shenv, results, timeoutHelper):
     builtin_commands = set(['cat'])
     builtin_commands_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "builtin_commands")
     # To avoid deadlock, we use a single stderr stream for piped
     # output. This is null until we have seen some output using
     # stderr.
     for i,j in enumerate(cmd.commands):
         # Reference the global environment by default.
         cmd_shenv = shenv
+        args = list(j.args)
         if j.args[0] == 'env':
             # Create a copy of the global environment and modify it for this one
             # command. There might be multiple envs in a pipeline:
             #   env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
             cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
-            updateEnv(cmd_shenv, j)
+            args = updateEnv(cmd_shenv, j.args)
 
         stdin, stdout, stderr = processRedirects(j, default_stdin, cmd_shenv,
                                                  opened_files)
 
         # If stderr wants to come from stdout, but stdout isn't a pipe, then put
         # stderr on a pipe and treat it as stdout.
         if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
             stderr = subprocess.PIPE
             stderrIsStdout = True
         else:
             stderrIsStdout = False
 
             # Don't allow stderr on a PIPE except for the last
             # process, this could deadlock.
             #
             # FIXME: This is slow, but so is deadlock.
             if stderr == subprocess.PIPE and j != cmd.commands[-1]:
                 stderr = tempfile.TemporaryFile(mode='w+b')
                 stderrTempFiles.append((i, stderr))
 
         # Resolve the executable path ourselves.
-        args = list(j.args)
         executable = None
         is_builtin_cmd = args[0] in builtin_commands;
         if not is_builtin_cmd:
             # For paths relative to cwd, use the cwd of the shell environment.
             if args[0].startswith('.'):
                 exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0])
                 if os.path.isfile(exe_in_cwd):
                     executable = exe_in_cwd
             if not executable:
                 executable = lit.util.which(args[0], cmd_shenv.env['PATH'])
             if not executable:
-                raise InternalShellError(j, '%r: command not found' % j.args[0])
+                raise InternalShellError(j, '%r: command not found' % args[0])
 
         # Replace uses of /dev/null with temporary files.
         if kAvoidDevNull:
             # In Python 2.x, basestring is the base class for all string (including unicode)
             # In Python 3.x, basestring no longer exist and str is always unicode
             try:
                 str_type = basestring
             except NameError:

llvm/utils/lit/tests/shtest-env.py

 # Check the env command
 #
-# RUN: %{lit} -a -v %{inputs}/shtest-env
+# RUN: %{lit} -j 1 -a -v %{inputs}/shtest-env \
+# RUN: | FileCheck -match-full-lines %s
+#
+# END.
+
+# Make sure env commands are included in printed commands.
+
+# CHECK: PASS: shtest-env :: env-u.txt ({{[^)]*}})
+# CHECK: $ "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "-u" "FOO" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "-u" "FOO" "-u" "BAR" "{{[^"]*}}" "print_environment.py"
+
+# CHECK: PASS: shtest-env :: env.txt ({{[^)]*}})
+# CHECK: $ "env" "A_FOO=999" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "A_FOO=1" "B_BAR=2" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
+
+# CHECK: PASS: shtest-env :: mixed.txt ({{[^)]*}})
+# CHECK: $ "env" "A_FOO=999" "-u" "FOO" "{{[^"]*}}" "print_environment.py"
+# CHECK: $ "env" "A_FOO=1" "-u" "FOO" "B_BAR=2" "-u" "BAR" "C_OOF=3" "{{[^"]*}}" "print_environment.py"
+
+# CHECK: Expected Passes : 3