[lit][test] Improve testing of use_llvm_tool

[jhenderson]

Fix some stale comments.

[jhenderson]

Fix another comment.

[MaskRay]

I just understand what D102630 and this do. Now rm $build/bin/ld.lld; llvm-lit something.s will not pick a ld.lld from PATH. Looks great!

curl -L 'https://reviews.llvm.org/D103154?download=1' | patch -p1
ninja check-lit

says use-llvm-tool.py fails.

[jhenderson]

In D103154#2783705, @MaskRay wrote:

I just understand what D102630 and this do. Now rm $build/bin/ld.lld; llvm-lit something.s will not pick a ld.lld from PATH. Looks great!

curl -L 'https://reviews.llvm.org/D103154?download=1' | patch -p1
ninja check-lit

says use-llvm-tool.py fails.

I stuffed up the diff somehow. The changes in config.py should be part of D103156, not here. I suspect that's what's breaking the test. Will investigate and fix.

[jhenderson]

Remove unrelated changes.

I'm still investigating the cause of the test failure - it doesn't fail for me locally on my windows machine.

[jhenderson]

In D103154#2784076, @jhenderson wrote:

Remove unrelated changes.

I'm still investigating the cause of the test failure - it doesn't fail for me locally on my windows machine.

The latest pre-merge test hasn't failed with this test as a failure, and the test passes on my Linux machine. I'm going to assume that the failure was just due to the broken diff and commit this as-is.

llvm/utils/lit/lit/llvm/config.py

def use_default_substitutions(self):
         self.config.substitutions.append(('%python', '"%s"' % (sys.executable)))
 
         self.add_tool_substitutions(
             tool_patterns, [self.config.llvm_tools_dir])
 
         self.add_err_msg_substitutions()
 
     def use_llvm_tool(self, name, search_env=None, required=False, quiet=False,
-                      use_installed=False):
+                      search_paths=None, use_installed=False):
         """Find the executable program 'name', optionally using the specified
         environment variable as an override before searching the build directory
         and then optionally the configuration's PATH."""
         # If the override is specified in the environment, use it without
         # validation.
         tool = None
         if search_env:
             tool = self.config.environment.get(search_env)
 
         if not tool:
-            # Use the build directory version.
-            tool = lit.util.which(name, self.config.llvm_tools_dir)
+            if search_paths is None:
+                search_paths = self.config.llvm_tools_dir
+            # Use the specified search paths.
+            path = os.pathsep.join(search_paths)
+            tool = lit.util.which(name, path)
 
         if not tool and use_installed:
             # Otherwise look in the path, if enabled.
             tool = lit.util.which(name, self.config.environment['PATH'])
 
         if required and not tool:
             message = "couldn't find '{}' program".format(name)
             if search_env:
def use_clang(self, additional_tool_dirs=[], additional_flags=[],
         if shl:
             self.config.substitutions.append(('%llvmshlibdir', shl))
         if pext:
             self.config.substitutions.append(('%pluginext', pext))
 
         # Discover the 'clang' and 'clangcc' to use.
         self.config.clang = self.use_llvm_tool(
             'clang', search_env='CLANG', required=required,
-            use_installed=use_installed)
+            search_paths=paths, use_installed=use_installed)
         if self.config.clang:
           self.config.available_features.add('clang')
           builtin_include_dir = self.get_clang_builtin_include_dir(
               self.config.clang)
           tool_substitutions = [
               ToolSubst('%clang', command=self.config.clang,
                         extra_args=additional_flags),
               ToolSubst('%clang_analyze_cc1', command='%clang_cc1',
def use_lld(self, additional_tool_dirs=[], required=True,
                          'lld_tools_dir', 'llvm_tools_dir']
         paths = [getattr(self.config, pp) for pp in exe_dir_props
                  if getattr(self.config, pp, None)]
         paths = additional_tool_dirs + paths
         self.with_environment('PATH', paths, append_path=True)
 
         lib_dir_props = [self.config.name.lower() + '_libs_dir',
                          'lld_libs_dir', 'llvm_libs_dir']
-        paths = [getattr(self.config, pp) for pp in lib_dir_props
+        lib_paths = [getattr(self.config, pp) for pp in lib_dir_props
                      if getattr(self.config, pp, None)]
 
-        self.with_environment('LD_LIBRARY_PATH', paths, append_path=True)
+        self.with_environment('LD_LIBRARY_PATH', lib_paths, append_path=True)
 
         # Discover the LLD executables to use.
 
         ld_lld = self.use_llvm_tool('ld.lld', required=required,
+                                    search_paths=paths,
                                     use_installed=use_installed)
         lld_link = self.use_llvm_tool('lld-link', required=required,
+                                      search_paths=paths,
                                       use_installed=use_installed)
         ld64_lld = self.use_llvm_tool('ld64.lld', required=required,
+                                      search_paths=paths,
                                       use_installed=use_installed)
         wasm_ld = self.use_llvm_tool('wasm-ld', required=required,
+                                     search_paths=paths,
                                      use_installed=use_installed)
 
         was_found = ld_lld and lld_link and ld64_lld and wasm_ld
         tool_substitutions = []
         if ld_lld:
             tool_substitutions.append(ToolSubst(r'ld\.lld', command=ld_lld))
             self.config.available_features.add('ld.lld')
         if lld_link:

llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg

+import lit.formats
+config.name = 'use-llvm-tool-required'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+import os.path
+config.llvm_tools_dir = os.path.realpath(os.path.dirname(__file__))
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+lit.llvm.llvm_config.use_llvm_tool('found', required=True)
+lit.llvm.llvm_config.use_llvm_tool('not-found', required=True)

llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg

+import lit.formats
+config.name = 'use-llvm-tool'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+import os.path
+this_dir = os.path.realpath(os.path.dirname(__file__))
+config.llvm_tools_dir = os.path.join(this_dir, 'build')
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+lit.llvm.llvm_config.with_environment('CASE1', os.path.join(this_dir, 'env-case1'))
+lit.llvm.llvm_config.with_environment('CASE6', os.path.join(this_dir, 'env-case6'))
+lit.llvm.llvm_config.with_environment('PATH', os.path.join(this_dir, 'path'), append_path=True)
+lit.llvm.llvm_config.use_llvm_tool('case1', search_env='CASE1')
+lit.llvm.llvm_config.use_llvm_tool('case2', search_env='CASE2')
+lit.llvm.llvm_config.use_llvm_tool('case3')
+lit.llvm.llvm_config.use_llvm_tool('case4', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case5')
+lit.llvm.llvm_config.use_llvm_tool('case6', search_env='CASE6', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case7', use_installed=True)
+lit.llvm.llvm_config.use_llvm_tool('case8', use_installed=True)

llvm/utils/lit/tests/Inputs/use-llvm-tool/true.txt

+# RUN: true

llvm/utils/lit/tests/Inputs/use-tool-search-env/lit.cfg

-import lit.formats
-config.name = 'search-env'
-config.suffixes = ['.txt']
-config.test_format = lit.formats.ShTest()
-config.test_source_root = None
-config.test_exec_root = None
-import lit.llvm
-lit.llvm.initialize(lit_config, config)
-import os.path
-path = os.path.realpath(os.path.join(os.path.dirname(__file__), 'test.tool'))
-lit.llvm.llvm_config.with_environment('TOOL_LOCATION', path)
-lit.llvm.llvm_config.use_llvm_tool('test-tool', search_env='TOOL_LOCATION')

llvm/utils/lit/tests/use-llvm-tool.py

+## Show that lit reports the path of tools found via use_llvm_tool.
+## Additionally show that use_llvm_tool uses in order of preference:
+## 1) The path specified in an environment variable,
+## 2) The LLVM tools build directory,
+## 3) The PATH, if requested.
+
+# RUN: %{lit} %{inputs}/use-llvm-tool 2>&1 | \
+# RUN:   FileCheck %s -DDIR=%p
+
+## The exact breakdown of cases is:
+## Case | Env | Build Dir | PATH |
+##   1  |  /  |     X     |  N/S | <- Can be found via env
+##   2  |  X  |     /     |  N/S | <- Can be found via build dir if env specified
+##   3  | N/S |     /     |  N/S | <- Can be found via build dir
+##   4  | N/S |     X     |   /  | <- Can be found via PATH, if requested
+##   5  | N/S |     X     |  N/S | <- Cannot be found via PATH, if not requested
+##   6  |  /  |     /     |   /  | <- Env is preferred over build, PATH
+##   7  | N/S |     /     |   /  | <- Build dir is preferred over PATH
+##   8  |  X  |     X     |   X  | <- Say nothing if cannot be found if not required
+
+## Check the exact path reported for the first case, but don't bother for the
+## others.
+# CHECK:      note: using case1: [[DIR]]{{[\\/]}}Inputs{{[\\/]}}use-llvm-tool{{[\\/]}}env-case1
+# CHECK-NEXT: note: using case2: {{.*}}build{{[\\/]}}case2
+# CHECK-NEXT: note: using case3: {{.*}}build{{[\\/]}}case3
+# CHECK-NEXT: note: using case4: {{.*}}path{{[\\/]}}case4
+# CHECK-NOT:  case5
+# CHECK-NEXT: note: using case6: {{.*}}env-case6
+# CHECK-NEXT: note: using case7: {{.*}}build{{[\\/]}}case7
+# CHECK-NOT:  case8
+
+## Test that if required is True, lit errors if the tool is not found.
+# RUN: not %{lit} %{inputs}/use-llvm-tool-required 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERROR
+# ERROR:      note: using found: {{.*}}found
+# ERROR-NEXT: fatal: couldn't find 'not-found' program

llvm/utils/lit/tests/use-tool-search-env.py

-## Show that lit reports the path to tools picked up via the use_llvm_tool
-## function when the tool is found via an environment variable.
-
-# RUN: %{lit} %{inputs}/use-tool-search-env 2>&1 | \
-# RUN:   FileCheck %s -DDIR=%p
-
-# CHECK: note: using test-tool: [[DIR]]{{[\\/]}}Inputs{{[\\/]}}use-tool-search-env{{[\\/]}}test.tool