[lit][test] Improve testing of use_llvm_tool

[jhenderson]

Fix some stale comments.

[jhenderson]

Fix another comment.

[MaskRay]

I just understand what D102630 and this do. Now rm $build/bin/ld.lld; llvm-lit something.s will not pick a ld.lld from PATH. Looks great!

curl -L 'https://reviews.llvm.org/D103154?download=1' | patch -p1
ninja check-lit

says use-llvm-tool.py fails.

[jhenderson]

In D103154#2783705, @MaskRay wrote:

I just understand what D102630 and this do. Now rm $build/bin/ld.lld; llvm-lit something.s will not pick a ld.lld from PATH. Looks great!

curl -L 'https://reviews.llvm.org/D103154?download=1' | patch -p1
ninja check-lit

says use-llvm-tool.py fails.

I stuffed up the diff somehow. The changes in config.py should be part of D103156, not here. I suspect that's what's breaking the test. Will investigate and fix.

[jhenderson]

Remove unrelated changes.

I'm still investigating the cause of the test failure - it doesn't fail for me locally on my windows machine.

[jhenderson]

In D103154#2784076, @jhenderson wrote:

Remove unrelated changes.

I'm still investigating the cause of the test failure - it doesn't fail for me locally on my windows machine.

The latest pre-merge test hasn't failed with this test as a failure, and the test passes on my Linux machine. I'm going to assume that the failure was just due to the broken diff and commit this as-is.

llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg

+import lit.formats
+config.name = 'use-llvm-tool-required'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+import os.path
+config.llvm_tools_dir = os.path.realpath(os.path.dirname(__file__))
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+lit.llvm.llvm_config.use_llvm_tool('found', required=True)
+lit.llvm.llvm_config.use_llvm_tool('not-found', required=True)

llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg

+import lit.formats
+config.name = 'use-llvm-tool'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+import os.path
+this_dir = os.path.realpath(os.path.dirname(__file__))
+config.llvm_tools_dir = os.path.join(this_dir, 'build')
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+lit.llvm.llvm_config.with_environment('CASE1', os.path.join(this_dir, 'env-case1'))
+lit.llvm.llvm_config.with_environment('CASE6', os.path.join(this_dir, 'env-case6'))
+lit.llvm.llvm_config.with_environment('PATH', os.path.join(this_dir, 'path'), append_path=True)
+lit.llvm.llvm_config.use_llvm_tool('case1', search_env='CASE1')
+lit.llvm.llvm_config.use_llvm_tool('case2', search_env='CASE2')
+lit.llvm.llvm_config.use_llvm_tool('case3')
+lit.llvm.llvm_config.use_llvm_tool('case4', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case5')
+lit.llvm.llvm_config.use_llvm_tool('case6', search_env='CASE6', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case7', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case8', use_installed=True)

llvm/utils/lit/tests/Inputs/use-tool-search-env/lit.cfg

-import lit.formats
-config.name = 'search-env'
-config.suffixes = ['.txt']
-config.test_format = lit.formats.ShTest()
-config.test_source_root = None
-config.test_exec_root = None
-import lit.llvm
-lit.llvm.initialize(lit_config, config)
-import os.path
-path = os.path.realpath(os.path.join(os.path.dirname(__file__), 'test.tool'))
-lit.llvm.llvm_config.with_environment('TOOL_LOCATION', path)
-lit.llvm.llvm_config.use_llvm_tool('test-tool', search_env='TOOL_LOCATION')

llvm/utils/lit/tests/use-llvm-tool.py

+## Show that lit reports the path of tools found via use_llvm_tool.
+## Additionally show that use_llvm_tool uses in order of preference:
+## 1) The path specified in an environment variable,
+## 2) The LLVM tools build directory,
+## 3) The PATH, if requested.
+
+# RUN: %{lit} %{inputs}/use-llvm-tool 2>&1 | \
+# RUN:   FileCheck %s -DDIR=%p
+
+## The exact breakdown of cases is:
+## Case | Env | Build Dir | PATH |
+##   1  |  /  |     X     |  N/S | <- Can be found via env
+##   2  |  X  |     /     |  N/S | <- Can be found via build dir if env specified
+##   3  | N/S |     /     |  N/S | <- Can be found via build dir
+##   4  | N/S |     X     |   /  | <- Can be found via PATH, if requested
+##   5  | N/S |     X     |  N/S | <- Cannot be found via PATH, if not requested
+##   6  |  /  |     /     |   /  | <- Env is preferred over build, PATH
+##   7  | N/S |     /     |   /  | <- Build dir is preferred over PATH
+##   8  |  X  |     X     |   X  | <- Say nothing if cannot be found if not required
+
+## Check the exact path reported for the first case, but don't bother for the
+## others.
+# CHECK:      note: using case1: [[DIR]]{{[\\/]}}Inputs{{[\\/]}}use-llvm-tool{{[\\/]}}env-case1
+# CHECK-NEXT: note: using case2: {{.*}}build{{[\\/]}}case2
+# CHECK-NEXT: note: using case3: {{.*}}build{{[\\/]}}case3
+# CHECK-NEXT: note: using case4: {{.*}}path{{[\\/]}}case4
+# CHECK-NOT:  case5
+# CHECK-NEXT: note: using case6: {{.*}}env-case6
+# CHECK-NEXT: note: using case7: {{.*}}build{{[\\/]}}case7
+# CHECK-NOT:  case8
+
+## Test that if required is True, lit errors if the tool is not found.
+# RUN: not %{lit} %{inputs}/use-llvm-tool-required 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERROR
+# ERROR:      note: using found: {{.*}}found
+# ERROR-NEXT: fatal: couldn't find 'not-found' program

llvm/utils/lit/tests/use-tool-search-env.py

-## Show that lit reports the path to tools picked up via the use_llvm_tool
-## function when the tool is found via an environment variable.
-
-# RUN: %{lit} %{inputs}/use-tool-search-env 2>&1 | \
-# RUN:   FileCheck %s -DDIR=%p
-
-# CHECK: note: using test-tool: [[DIR]]{{[\\/]}}Inputs{{[\\/]}}use-tool-search-env{{[\\/]}}test.tool