[lldb] [test] Fix finding LLDB tools when building stand-alone

[mgorny]

Summary

Use both LLDB- and LLVM-specific tool/library directories when LLDB is
being built stand-alone. This ensures that the freshly-built tools
(and libraries) are used correctly.

Without this patch, the test suite uses LLVM_TOOLS_DIR and LLVM_LIBS_DIR
to locate lldb, and set PATH and LD_LIBRARY_PATH. When doing
a stand-alone build, these variables represent the installed LLVM.
As a result, tests either fail due to missing lldb executable
or use an earlier installed LLDB version rather than the one being
built.

To solve this, additional LLDB_TOOLS_DIR and LLDB_LIBS_DIR variables
are added and populated using LLVM_*_OUTPUT_INTDIR. Those variables
contain directories used to output built executables and libraries.
In stand-alone builds, they represent the build-tree directories
used by LLDB. In integrated builds, they have the same values as
LLVM_*_DIR and therefore using them does not harm.

The new variables are prepended to PATH and LD_LIBRARY_PATH to ensure
that freshly built binaries are preferred over potentially earlier
installed ones. Furthermore, paths used to locate various tools are
updated to match appropriate locations.

[labath]

It does not seem too controversial, but I am not very confident reviewing lit changes. @beanz, could you take a look at this?

lit/lit.cfg

124

Wouldn't it be better to use lldb_tools_dir instead of the PATH here? Less chance to pick up the wrong tool accidentally?

[mgorny]

Following @labath's suggestion, I've changed my approach a little. Now I'm using PATH lookup only for CC/CXX, while for remaining tools I'm using either llvm_tools_dir or lldb_tools_dir appropriately.

[labath]

Thanks, I feel more comfortable with this one.

Let's give the others a but more time to chime in. If they don't I guess you can submit this. :)

lit/lit.cfg

if lldb_obj_root is not None:
     config.test_exec_root = os.path.join(lldb_obj_root, 'lit')
 
 # Set llvm_{src,obj}_root for use by others.
 config.llvm_src_root = getattr(config, 'llvm_src_root', None)
 config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)
 
 # Tweak the PATH to include the tools dir and the scripts dir.
 if lldb_obj_root is not None:
+    lldb_tools_dir = getattr(config, 'lldb_tools_dir', None)
+    if not lldb_tools_dir:
+        lit_config.fatal('No LLDB tools dir set!')
     llvm_tools_dir = getattr(config, 'llvm_tools_dir', None)
     if not llvm_tools_dir:
         lit_config.fatal('No LLVM tools dir set!')
-    path = os.path.pathsep.join((llvm_tools_dir, config.environment['PATH']))
+    path = os.path.pathsep.join((lldb_tools_dir, llvm_tools_dir, config.environment['PATH']))
     path = os.path.pathsep.join((os.path.join(getattr(config, 'llvm_src_root', None),'test','Scripts'),path))
 
     config.environment['PATH'] = path
 
+    lldb_libs_dir = getattr(config, 'lldb_libs_dir', None)
+    if not lldb_libs_dir:
+        lit_config.fatal('No LLDB libs dir set!')
     llvm_libs_dir = getattr(config, 'llvm_libs_dir', None)
     if not llvm_libs_dir:
         lit_config.fatal('No LLVM libs dir set!')
-    path = os.path.pathsep.join((llvm_libs_dir,
+    path = os.path.pathsep.join((lldb_libs_dir, llvm_libs_dir,
                                  config.environment.get('LD_LIBRARY_PATH','')))
     config.environment['LD_LIBRARY_PATH'] = path
 
     # Propagate LLVM_SRC_ROOT into the environment.
     config.environment['LLVM_SRC_ROOT'] = getattr(config, 'llvm_src_root', '')
 
     # Propagate PYTHON_EXECUTABLE into the environment
     config.environment['PYTHON_EXECUTABLE'] = getattr(config, 'python_executable',
if config.test_exec_root is None:
     # Okay, that worked. Notify the user of the automagic, and reconfigure.
     lit_config.note('using out-of-tree build at %r' % lldb_obj_root)
     lit_config.load_config(config, site_cfg)
     raise SystemExit
 
 # Register substitutions
 config.substitutions.append(('%python', config.python_executable))
 
-debugserver = lit.util.which('debugserver', llvm_tools_dir)
-lldb = lit.util.which('lldb', llvm_tools_dir)
+debugserver = lit.util.which('debugserver', config.environment['PATH'])
+lldb = lit.util.which('lldb', config.environment['PATH'])
 
 if not os.path.exists(config.cc):
-    config.cc = lit.util.which(config.cc, llvm_tools_dir)
+    config.cc = lit.util.which(config.cc, config.environment['PATH'])
 
 if not os.path.exists(config.cxx):
-    config.cxx = lit.util.which(config.cxx, llvm_tools_dir)
+    config.cxx = lit.util.which(config.cxx, config.environment['PATH'])
 
 if platform.system() in ['Darwin']:
     try:
         out = lit.util.capture(['xcrun', '--show-sdk-path']).strip()
         res = 0
     except OSError:
         res = -1
     if res == 0 and out:
if debugserver is not None:
     config.substitutions.append(('%debugserver', debugserver))
 
 for pattern in [r"\bFileCheck\b",
                 r"\| \bnot\b"]:
     tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
                           pattern)
     tool_pipe = tool_match.group(2)
     tool_name = tool_match.group(4)
-    tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
+    tool_path = lit.util.which(tool_name, config.environment['PATH'])
     if not tool_path:
         # Warn, but still provide a substitution.
         lit_config.note(
-            'Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
+            'Did not find ' + tool_name + ' in ' + config.environment['PATH'])
     config.substitutions.append((pattern, tool_pipe + tool_path))
 
 # Shell execution
 if platform.system() not in ['Windows'] or lit_config.getBashPath() != '':
     config.available_features.add('shell')
 
 # Running on Darwin OS
 if platform.system() in ['Darwin']:

 elif re.match(r'gcc', config.cc):
     config.available_features.add("compiler-gcc")
 elif re.match(r'cl', config.cc):
     config.available_features.add("compiler-msvc")
 
 # llvm-config knows whether it is compiled with asserts (and)
 # whether we are operating in release/debug mode.
 import subprocess
+llvm_config = lit.util.which('llvm-config', config.environment['PATH'])
+if not llvm_config:
+    lit_config.fatal("Could not find llvm-config in " + config.environment['PATH'])
 try:
     llvm_config_cmd = \
-     subprocess.Popen([os.path.join(llvm_tools_dir, 'llvm-config'),
+     subprocess.Popen([llvm_config,
                      '--build-mode', '--assertion-mode', '--targets-built'],
                       stdout = subprocess.PIPE)
 except OSError as why:
-    print("Could not find llvm-config in " + llvm_tools_dir)
+    print("llvm-config invocation failed")
     exit(42)
 
 llvm_config_output = llvm_config_cmd.stdout.read().decode('utf_8')
 llvm_config_output_list = llvm_config_output.split("\n")
 
 if re.search(r'DEBUG', llvm_config_output_list[0]):
     config.available_features.add('debug')
 if re.search(r'ON', llvm_config_output_list[1]):
     config.available_features.add('asserts')
 if re.search(r'ARM', llvm_config_output_list[2]):
     config.available_features.add('arm')
 if re.search(r'Mips', llvm_config_output_list[2]):
     config.available_features.add('mips')
 if re.search(r'X86', llvm_config_output_list[2]):
     config.available_features.add('x86')
 llvm_config_cmd.wait()

lit/lit.site.cfg.in

 @LIT_SITE_CFG_IN_HEADER@
 
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
 config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.lldb_obj_root = "@LLDB_BINARY_DIR@"
+config.lldb_libs_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
+config.lldb_tools_dir = "@LLVM_RUNTIME_OUTPUT_INTDIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.cc = "@CMAKE_C_COMPILER@"
 config.cxx = "@CMAKE_CXX_COMPILER@"
 
 test_c_compiler = "@LLDB_TEST_C_COMPILER@"
 test_cxx_compiler = "@LLDB_TEST_CXX_COMPILER@"
 test_clang = "@LLDB_TEST_CLANG@".lower()