Index: llvm/test/FileCheck/lit.local.cfg =================================================================== --- llvm/test/FileCheck/lit.local.cfg +++ llvm/test/FileCheck/lit.local.cfg @@ -1,13 +1,61 @@ -# Unset environment variables that the FileCheck tests -# expect not to be set. -file_check_expected_unset_vars = [ - 'FILECHECK_DUMP_INPUT_ON_FAILURE', - 'FILECHECK_OPTS', -] - -for env_var in file_check_expected_unset_vars: - if env_var in config.environment: - lit_config.note('Removing {} from environment for FileCheck tests'.format( - env_var) - ) - config.environment.pop(env_var) +# There are a few main points to remember about %TestFileCheckOut: +# +# 1. %TestFileCheckOut should precede a FileCheck call in FileCheck's test +# suite if and only if the call's textual output affects test results. +# If only the call's exit status matters, don't use %TestFileCheckOut. +# 2. %TestFileCheckOut expands to an "env" command that unsets all FileCheck +# environment variables. Thus, any FileCheck environment variables that +# need to be tested must be assigned between %TestFileCheckOut and the +# FileCheck command. +# 3. %TestFileCheckOut's purpose is to ensure correct test results when +# developers set FileCheck environment variables (e.g., +# FILECHECK_OPTS=-dump-input=fail) to tweak the output of FileCheck while +# debugging tests. If a developer sets values that affect FileCheck's exit +# status (e.g., FILECHECK_OPTS=-strict-whitespace), he shouldn't be +# surprised that test results throughout all test suites are affected. +# +# Examples: +# +# ; Test another program, using FileCheck to verify its textual output. +# ; Only FileCheck's exit status affects test results, so a bare FileCheck +# ; call is sufficient and more convenient for debugging. +# ; +# ; RUN: %t | FileCheck %s +# ; CHECK: {{[0-9]+\.0}} +# +# ; Test FileCheck itself, but only examine its exit status, so a bare +# ; FileCheck call is still sufficient and more convenient for debugging. +# ; +# ; RUN: FileCheck -input-file %s %s +# ; CHECK: {{[0-9]+\.0}} +# ; 5.0 +# +# ; Check that the FileCheck trace is off by default. The first FileCheck +# ; call's textual output affects test results, so it requires +# ; %TestFileCheckOut to be safe. +# ; +# ; RUN: %TestFileCheckOut FileCheck -input-file %s %s 2>&1 \ +# ; RUN: | FileCheck -allow-empty -check-prefix QUIET %s +# ; +# ; CHECK: {{[0-9]+\.0}} +# ; 5.0 +# ; QUIET-NOT: expected string found in input +# +# ; Check that the FileCheck trace is on when FILECHECK_OPTS=-v. +# ; FILECHECK_OPTS must be set after %TestFileCheckOut, which clears +# ; FILECHECK_OPTS beforehand. +# ; +# ; RUN: %TestFileCheckOut FILECHECK_OPTS=-v \ +# ; RUN: FileCheck -input-file %s %s 2>&1 \ +# ; RUN: | FileCheck -check-prefix TRACE %s +# ; +# ; CHECK: {{[0-9]+\.0}} +# ; 5.0 +# ; TRACE: expected string found in input +# +# The name %TestFileCheckOut is verbose. However, %TestFileCheckOut is +# unlikely to prove useful in other test suites, and most developers don't +# spend much time in FileCheck's test suite. A strong reminder of its intended +# usage at every occurrence seems worth the verbosity. +config.substitutions.append(('%TestFileCheckOut', + 'env -u FILECHECK_OPTS -u FILECHECK_DUMP_INPUT_ON_FAILURE')) Index: llvm/utils/lit/tests/lit.cfg =================================================================== --- llvm/utils/lit/tests/lit.cfg +++ llvm/utils/lit/tests/lit.cfg @@ -43,10 +43,17 @@ if directory: llvm_config.with_environment('PATH', directory, append_path=True) +# This test suite calls %{lit} to test lit's behavior for the sample test +# suites in %{inputs}. This test suite's results are then determined in part +# by %{lit}'s textual output, which includes the output of FileCheck calls +# within %{inputs}'s test suites. Thus, %{lit} clears environment variables +# that can affect FileCheck's output. config.substitutions.append(('%{inputs}', os.path.join( config.test_source_root, 'Inputs'))) -config.substitutions.append(('%{lit}', "%%{python} %s" % ( - os.path.join(lit_path, 'lit.py'),))) +config.substitutions.append(('%{lit}', + "{env} %{{python}} {lit}".format( + env="env -u FILECHECK_OPTS -u FILECHECK_DUMP_INPUT_ON_FAILURE", + lit=os.path.join(lit_path, 'lit.py')))) config.substitutions.append(('%{python}', '"%s"' % (sys.executable))) # Enable coverage.py reporting, assuming the coverage module has been installed Index: llvm/utils/lit/tests/shtest-run-at-line.py =================================================================== --- llvm/utils/lit/tests/shtest-run-at-line.py +++ llvm/utils/lit/tests/shtest-run-at-line.py @@ -1,7 +1,7 @@ # Check that -vv makes the line number of the failing RUN command clear. # (-v is actually sufficient in the case of the internal shell.) # -# RUN: env -u FILECHECK_OPTS not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out +# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out # RUN: FileCheck --input-file %t.out %s # # END.