Allow use of self.filecheck in LLDB tests (c.f self.expect)

[vsk]

Summary

Add a "filecheck" method to the LLDB test base. This allows test authors
to pattern match command output using FileCheck, making it possible to
write stricter tests than what self.expect allows.

For context (motivation, examples of stricter checking, etc), see the
lldb-dev thread: "Using FileCheck in lldb inline tests".

[teemperor]

lldb/packages/Python/lldbsuite/test/configuration.py

197

This is true if we build LLDB as part of LLVM, but this doesn't seem correct if LLDB is built against an existing LLVM (which is in general supported in LLDB's CMake if IIUC). I think the best way is to pass the binary directory for this to dotest and configure it in the lit.cfg.

207

Windows itself can probably handle the fact that the .exe suffix is missing for FileCheck, but this assert will fail in this situation.

lldb/packages/Python/lldbsuite/test/lldbtest.py

51

I think that list is supposed to be sorted alphabetically

2166

Making a temporary file and opening it later by name is not really supported (see the NamedTemporaryFile docs). Could we maybe pass the output to FileCheck via stdin? That would also save us from the detour over the fs.

2181

We should quote the paths in here. Or at least, the temporary file path should be quoted as it will contain spaces on some setups.

[stella.stamenova]

lldb/packages/Python/lldbsuite/test/configuration.py

197

Rather than making the change here, as @teemperor pointed out, you should use the existing mechanisms we have in the CMake files. Look for LLDB_DEFAULT_TEST_DSYMUTIL to see how we handle this (and the extension too!) for dsymutil.

[vsk]

Sorry for the delay, I was busy with other work. I think I've addressed the review feedback. PTAL.

[stella.stamenova]

LGTM, but I'd wait till @teemperor approves as well.

[JDevlieghere]

Thanks Vedant, this looks very useful!

[teemperor]

LGTM minus one small detail (see inline comment). Thanks Vedant!

lldb/packages/Python/lldbsuite/test/lldbtest.py

51

I think we don't need that import anymore.

lldb/CMakeLists.txt

 add_subdirectory(tools)
 
 option(LLDB_INCLUDE_TESTS "Generate build targets for the LLDB unit tests."
   ${LLVM_INCLUDE_TESTS})
 option(LLDB_TEST_USE_CUSTOM_C_COMPILER "Use the C compiler provided via LLDB_TEST_C_COMPILER for building test inferiors (instead of the just-built compiler). Defaults to OFF." OFF)
 option(LLDB_TEST_USE_CUSTOM_CXX_COMPILER "Use the C++ compiler provided via LLDB_TEST_CXX_COMPILER for building test inferiors (instead of the just-built compiler). Defaults to OFF." OFF)
 if(LLDB_INCLUDE_TESTS)
 
-  # The difference between the following two paths is significant. The path to
-  # LLDB will point to LLDB's binary directory, while the other will point to
-  # LLVM's binary directory in case the two differ.
+  # Set the path to the default lldb test executable. Make the path relative to
+  # LLVM_RUNTIME_OUTPUT_INTDIR: this will be correct even when LLVM and LLDB
+  # have separate binary directories.
   set(LLDB_DEFAULT_TEST_EXECUTABLE "${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb${CMAKE_EXECUTABLE_SUFFIX}")
+
+  # Set the paths to default llvm tools. Make these paths relative to the LLVM
+  # binary directory.
   set(LLDB_DEFAULT_TEST_DSYMUTIL "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/dsymutil${CMAKE_EXECUTABLE_SUFFIX}")
+  set(LLDB_DEFAULT_TEST_FILECHECK "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/FileCheck${CMAKE_EXECUTABLE_SUFFIX}")
 
   if (NOT LLDB_TEST_USE_CUSTOM_C_COMPILER AND TARGET clang)
     set(LLDB_DEFAULT_TEST_C_COMPILER "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/clang${CMAKE_EXECUTABLE_SUFFIX}")
   else()
     set(LLDB_DEFAULT_TEST_C_COMPILER "")
   endif()
 
   if (NOT LLDB_TEST_USE_CUSTOM_CXX_COMPILER AND TARGET clang)
     set(LLDB_DEFAULT_TEST_CXX_COMPILER "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/clang++${CMAKE_EXECUTABLE_SUFFIX}")
   else()
     set(LLDB_DEFAULT_TEST_CXX_COMPILER "")
   endif()
 
   set(LLDB_TEST_EXECUTABLE "${LLDB_DEFAULT_TEST_EXECUTABLE}" CACHE PATH "lldb executable used for testing")
   set(LLDB_TEST_C_COMPILER "${LLDB_DEFAULT_TEST_C_COMPILER}" CACHE PATH "C Compiler to use for building LLDB test inferiors")
   set(LLDB_TEST_CXX_COMPILER "${LLDB_DEFAULT_TEST_CXX_COMPILER}" CACHE PATH "C++ Compiler to use for building LLDB test inferiors")
   set(LLDB_TEST_DSYMUTIL "${LLDB_DEFAULT_TEST_DSYMUTIL}" CACHE PATH "dsymutil used for generating dSYM bundles")
+  set(LLDB_TEST_FILECHECK "${LLDB_DEFAULT_TEST_FILECHECK}" CACHE PATH "FileCheck used for testing purposes")
 
   if (("${LLDB_TEST_C_COMPILER}" STREQUAL "") OR
       ("${LLDB_TEST_CXX_COMPILER}" STREQUAL ""))
     message(FATAL_ERROR "LLDB test compilers not specified.  Tests will not run")
   endif()
 
   set(LLDB_TEST_DEPS lldb)
 

lldb/lldb.xcodeproj/project.pbxproj

2689FFC813353D7A00698AC0 /* Headers */ = {
       );
       runOnlyForDeploymentPostprocessing = 0;
     };
 /* End PBXHeadersBuildPhase section */
 
 /* Begin PBXLegacyTarget section */
     2387551E1C24974600CCE8C3 /* lldb-python-test-suite */ = {
       isa = PBXLegacyTarget;
-      buildArgumentsString = "-u $(SRCROOT)/test/dotest.py --apple-sdk $(PLATFORM_NAME) --executable=$(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)/lldb -C $(LLDB_PYTHON_TESTSUITE_CC) --arch $(LLDB_PYTHON_TESTSUITE_ARCH) --session-file-format fm --results-formatter lldbsuite.test_event.formatter.xunit.XunitFormatter --build-dir $(BUILD_DIR)/lldb-test-build.noindex --results-file $(BUILD_DIR)/test-results-$(LLDB_PYTHON_TESTSUITE_ARCH).xml --rerun-all-issues --env TERM=vt100 -O--xpass=ignore";
+      buildArgumentsString = "-u $(SRCROOT)/test/dotest.py --apple-sdk $(PLATFORM_NAME) --executable=$(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)/lldb -C $(LLDB_PYTHON_TESTSUITE_CC) --arch $(LLDB_PYTHON_TESTSUITE_ARCH) --filecheck $(LLVM_BUILD_DIR)/x86_64/bin/FileCheck --session-file-format fm --results-formatter lldbsuite.test_event.formatter.xunit.XunitFormatter --build-dir $(BUILD_DIR)/lldb-test-build.noindex --results-file $(BUILD_DIR)/test-results-$(LLDB_PYTHON_TESTSUITE_ARCH).xml --rerun-all-issues --env TERM=vt100 -O--xpass=ignore";
       buildConfigurationList = 238755241C24974600CCE8C3 /* Build configuration list for PBXLegacyTarget "lldb-python-test-suite" */;
       buildPhases = (
       );
       buildToolPath = /usr/bin/python;
       buildWorkingDirectory = "$(BUILD_DIR)";
       dependencies = (
       );
       name = "lldb-python-test-suite";

lldb/packages/Python/lldbsuite/test/configuration.py

 
 # Test suite repeat count.  Can be overwritten with '-# count'.
 count = 1
 
 # The 'arch' and 'compiler' can be specified via command line.
 arch = None        # Must be initialized after option parsing
 compiler = None    # Must be initialized after option parsing
 
+# Path to the FileCheck testing tool. Not optional.
+filecheck = None
+
 # The arch might dictate some specific CFLAGS to be passed to the toolchain to build
 # the inferior programs.  The global variable cflags_extras provides a hook to do
 # just that.
 cflags_extras = ''
 
 # The filters (testclass.testmethod) used to admit tests into our test suite.
 filters = []
 
def get_absolute_path_to_root_test_dir():
     If an exclusive test subdirectory is specified, return its absolute path.
     Otherwise, return the absolute path of the root test directory.
     """
     test_subdir = get_absolute_path_to_exclusive_test_subdir()
     if test_subdir:
         return test_subdir
 
     return os.path.dirname(os.path.realpath(__file__))
+
+
+def get_filecheck_path():
+    """
+    Get the path to the FileCheck testing tool.
+    """
+    assert os.path.lexists(filecheck)
+    return filecheck

lldb/packages/Python/lldbsuite/test/dotest.py

else:
                     break
 
     if args.dsymutil:
       os.environ['DSYMUTIL'] = args.dsymutil
     elif platform_system == 'Darwin':
       os.environ['DSYMUTIL'] = seven.get_command_output(
           'xcrun -find -toolchain default dsymutil')
 
+    if args.filecheck:
+        # The CMake build passes in a path to a working FileCheck binary.
+        configuration.filecheck = os.path.abspath(args.filecheck)
+    else:
+        logging.error('No valid FileCheck executable; aborting...')
+        sys.exit(-1)
+
     if args.channels:
         lldbtest_config.channels = args.channels
 
     if args.log_success:
         lldbtest_config.log_success = args.log_success
 
     if args.out_of_tree_debugserver:
         lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver

lldb/packages/Python/lldbsuite/test/dotest_args.py

def create_parser():
     group.add_argument(
         '-E',
         metavar='extra-flags',
         help=textwrap.dedent('''Specify the extra flags to be passed to the toolchain when building the inferior programs to be debugged
                                                            suggestions: do not lump the "-A arch1 -A arch2" together such that the -E option applies to only one of the architectures'''))
 
     group.add_argument('--dsymutil', metavar='dsymutil', dest='dsymutil', help=textwrap.dedent('Specify which dsymutil to use.'))
 
+    group.add_argument('--filecheck', metavar='filecheck', dest='filecheck', help=textwrap.dedent('Specify which FileCheck binary to use.'))
+
     # Test filtering options
     group = parser.add_argument_group('Test filtering options')
     group.add_argument(
         '-f',
         metavar='filterspec',
         action='append',
         help='Specify a filter, which consists of the test class name, a dot, followed by the test method, to only admit such test into the test suite')  # FIXME: Example?
     X('-l', "Don't skip long running tests")

lldb/packages/Python/lldbsuite/test/expression_command/formatters/TestFormatters.py

def test(self):
         self.runCmd("command script import formatters.py")
         self.runCmd("command script import foosynth.py")
 
         if self.TraceOn():
             self.runCmd("frame variable foo1 --show-types")
             self.runCmd("frame variable foo1.b --show-types")
             self.runCmd("frame variable foo1.b.b_ref --show-types")
 
-        self.expect(
-            "expression --show-types -- *(new foo(47))",
-            substrs=[
-                '(int) a = 47',
-                '(bar) b = {',
-                '(int) i = 94',
-                '(baz) b = {',
-                '(int) k = 99'])
+        self.filecheck("expression --show-types -- *(new foo(47))", __file__,
+                '-check-prefix=EXPR-TYPES-NEW-FOO')
+        # EXPR-TYPES-NEW-FOO: (foo) ${{.*}} = {
+        # EXPR-TYPES-NEW-FOO-NEXT:   (int) a = 47
+        # EXPR-TYPES-NEW-FOO-NEXT:   (int *) a_ptr = 0x
+        # EXPR-TYPES-NEW-FOO-NEXT:   (bar) b = {
+        # EXPR-TYPES-NEW-FOO-NEXT:     (int) i = 94
+        # EXPR-TYPES-NEW-FOO-NEXT:     (int *) i_ptr = 0x
+        # EXPR-TYPES-NEW-FOO-NEXT:     (baz) b = {
+        # EXPR-TYPES-NEW-FOO-NEXT:       (int) h = 97
+        # EXPR-TYPES-NEW-FOO-NEXT:       (int) k = 99
+        # EXPR-TYPES-NEW-FOO-NEXT:     }
+        # EXPR-TYPES-NEW-FOO-NEXT:     (baz &) b_ref = 0x
+        # EXPR-TYPES-NEW-FOO-NEXT:   }
+        # EXPR-TYPES-NEW-FOO-NEXT: }
 
         self.runCmd("type summary add -F formatters.foo_SummaryProvider foo")
 
         self.expect("expression new int(12)",
                     substrs=['(int *) $', ' = 0x'])
 
         self.runCmd(
             "type summary add -s \"${var%pointer} -> ${*var%decimal}\" \"int *\"")
 
         self.expect("expression new int(12)",
                     substrs=['(int *) $', '= 0x', ' -> 12'])
 
         self.expect("expression foo1.a_ptr",
                     substrs=['(int *) $', '= 0x', ' -> 13'])
 
-        self.expect(
-            "expression foo1",
-            substrs=[
-                '(foo) $',
-                ' a = 12',
-                'a_ptr = ',
-                ' -> 13',
-                'i = 24',
-                'i_ptr = ',
-                ' -> 25'])
-
-        self.expect(
-            "expression --ptr-depth=1 -- new foo(47)",
-            substrs=[
-                '(foo *) $',
-                'a = 47',
-                'a_ptr = ',
-                ' -> 48',
-                'i = 94',
-                'i_ptr = ',
-                ' -> 95'])
-
-        self.expect(
-            "expression foo2",
-            substrs=[
-                '(foo) $',
-                'a = 121',
-                'a_ptr = ',
-                ' -> 122',
-                'i = 242',
-                'i_ptr = ',
-                ' -> 243'])
+        self.filecheck("expression foo1", __file__, '-check-prefix=EXPR-FOO1')
+        # EXPR-FOO1: (foo) $
+        # EXPR-FOO1-SAME: a = 12
+        # EXPR-FOO1-SAME: a_ptr = {{[0-9]+}} -> 13
+        # EXPR-FOO1-SAME: i = 24
+        # EXPR-FOO1-SAME: i_ptr = {{[0-9]+}} -> 25
+        # EXPR-FOO1-SAME: b_ref = {{[0-9]+}}
+        # EXPR-FOO1-SAME: h = 27
+        # EXPR-FOO1-SAME: k = 29
+
+        self.filecheck("expression --ptr-depth=1 -- new foo(47)", __file__,
+                '-check-prefix=EXPR-PTR-DEPTH1')
+        # EXPR-PTR-DEPTH1: (foo *) $
+        # EXPR-PTR-DEPTH1-SAME: a = 47
+        # EXPR-PTR-DEPTH1-SAME: a_ptr = {{[0-9]+}} -> 48
+        # EXPR-PTR-DEPTH1-SAME: i = 94
+        # EXPR-PTR-DEPTH1-SAME: i_ptr = {{[0-9]+}} -> 95
+
+        self.filecheck("expression foo2", __file__, '-check-prefix=EXPR-FOO2')
+        # EXPR-FOO2: (foo) $
+        # EXPR-FOO2-SAME: a = 121
+        # EXPR-FOO2-SAME: a_ptr = {{[0-9]+}} -> 122
+        # EXPR-FOO2-SAME: i = 242
+        # EXPR-FOO2-SAME: i_ptr = {{[0-9]+}} -> 243
+        # EXPR-FOO2-SAME: h = 245
+        # EXPR-FOO2-SAME: k = 247
 
         object_name = self.res.GetOutput()
         object_name = object_name[7:]
         object_name = object_name[0:object_name.find(' =')]
 
-        self.expect(
-            "frame variable foo2",
-            substrs=[
-                '(foo)',
-                'foo2',
-                'a = 121',
-                'a_ptr = ',
-                ' -> 122',
-                'i = 242',
-                'i_ptr = ',
-                ' -> 243'])
-
-        self.expect(
-            "expression $" +
-            object_name,
-            substrs=[
-                '(foo) $',
-                'a = 121',
-                'a_ptr = ',
-                ' -> 122',
-                'i = 242',
-                'i_ptr = ',
-                ' -> 243',
-                'h = 245',
-                'k = 247'])
+        self.filecheck("frame variable foo2", __file__, '-check-prefix=VAR-FOO2')
+        # VAR-FOO2: (foo) foo2
+        # VAR-FOO2-SAME: a = 121
+        # VAR-FOO2-SAME: a_ptr = {{[0-9]+}} -> 122
+        # VAR-FOO2-SAME: i = 242
+        # VAR-FOO2-SAME: i_ptr = {{[0-9]+}} -> 243
+        # VAR-FOO2-SAME: h = 245
+        # VAR-FOO2-SAME: k = 247
+
+        # The object is the same as foo2, so use the EXPR-FOO2 checks.
+        self.filecheck("expression $" + object_name, __file__,
+                '-check-prefix=EXPR-FOO2')
 
         self.runCmd("type summary delete foo")
         self.runCmd(
             "type synthetic add --python-class foosynth.FooSyntheticProvider foo")
 
         self.expect("expression --show-types -- $" + object_name,
                     substrs=['(foo) $', ' = {', '(int) *i_ptr = 243'])
 

lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp

 //===-- main.cpp --------------------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 typedef int Foo;
 
 int main() {
+    // CHECK: (Foo [3]) array = {
+    // CHECK-NEXT: (Foo) [0] = 1
+    // CHECK-NEXT: (Foo) [1] = 2
+    // CHECK-NEXT: (Foo) [2] = 3
+    // CHECK-NEXT: }
     Foo array[3] = {1,2,3};
-    return 0; //% self.expect("frame variable array --show-types --", substrs = ['(Foo [3]) array = {','(Foo) [0] = 1','(Foo) [1] = 2','(Foo) [2] = 3'])
+    return 0; //% self.filecheck("frame variable array --show-types --", 'main.cpp')
 }

lldb/packages/Python/lldbsuite/test/lldbtest.py

 import inspect
 import io
 import os.path
 import re
 import shutil
 import signal
 from subprocess import *
 import sys
+import tempfile
 import time
 import traceback
 import types
 import distutils.spawn
 
 # Third-party modules
 import unittest2
 from six import add_metaclass
def check_completion_with_desc(self, str_input, match_desc_pairs):
             found_pair = False
             for i in range(num_matches + 1):
                 match_candidate = match_strings.GetStringAtIndex(i)
                 description_candidate = description_strings.GetStringAtIndex(i)
                 if match_candidate == pair[0] and description_candidate == pair[1]:
                     found_pair = True
                     break
             if not found_pair:
                 missing_pairs.append(pair)
 
         if len(missing_pairs):
             error_msg = "Missing pairs:\n"
             for pair in missing_pairs:
                 error_msg += " [" + pair[0] + ":" + pair[1] + "]\n"
             error_msg += "Got the following " + str(num_matches) + " completions back:\n"
             for i in range(num_matches + 1):
                 match_candidate = match_strings.GetStringAtIndex(i)
                 description_candidate = description_strings.GetStringAtIndex(i)
                 error_msg += "[" + match_candidate + ":" + description_candidate + "]\n"
             self.assertEqual(0, len(missing_pairs), error_msg)
 
     def complete_exactly(self, str_input, patterns):
         self.complete_from_to(str_input, patterns, True)
 
     def complete_from_to(self, str_input, patterns, turn_off_re_match=False):
         """Test that the completion mechanism completes str_input to patterns,
         where patterns could be a pattern-string or a list of pattern-strings"""
         # Patterns should not be None in order to proceed.
         self.assertFalse(patterns is None)
         # And should be either a string or list of strings.  Check for list type
         # below, if not, make a list out of the singleton string.  If patterns
         # is not a string or not a list of strings, there'll be runtime errors
def complete_from_to(self, str_input, patterns, turn_off_re_match=False):
                 self.expect(
                     compare_string, msg=COMPLETION_MSG(
                         str_input, p, match_strings), exe=False, substrs=[p])
             else:
                 self.expect(
                     compare_string, msg=COMPLETION_MSG(
                         str_input, p, match_strings), exe=False, patterns=[p])
 
+    def filecheck(
+            self,
+            command,
+            check_file,
+            filecheck_options = ''):
+        # Run the command.
+        self.runCmd(
+                command,
+                msg="FileCheck'ing result of `{0}`".format(command))
+
+        # Get the error text if there was an error, and the regular text if not.
+        output = self.res.GetOutput() if self.res.Succeeded() \
+                else self.res.GetError()
+
+        # Assemble the absolute path to the check file. As a convenience for
+        # LLDB inline tests, assume that the check file is a relative path to
+        # a file within the inline test directory.
+        if check_file.endswith('.pyc'):
+            check_file = check_file[:-1]
+        if hasattr(self, 'test_filename'):
+            check_file_abs = os.path.join(os.path.dirname(self.test_filename),
+                    check_file)
+        else:
+            check_file_abs = os.path.abspath(check_file)
+
+        # Run FileCheck.
+        filecheck_bin = configuration.get_filecheck_path()
+        filecheck_args = [filecheck_bin, check_file_abs]
+        if filecheck_options:
+            filecheck_args.append(filecheck_options)
+        subproc = Popen(filecheck_args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        cmd_stdout, cmd_stderr = subproc.communicate(input=output)
+        cmd_status = subproc.returncode
+
+        if cmd_status != 0:
+            filecheck_cmd = " ".join(filecheck_args)
+            self.assertTrue(cmd_status == 0, """
+--- FileCheck failed (code={0}) ---
+{1}
+
+FileCheck input:
+{2}
+
+FileCheck output:
+{3}
+{4}
+""".format(cmd_status, filecheck_cmd, output, cmd_stdout, cmd_stderr))
+
     def expect(
             self,
             str,
             msg=None,
             patterns=None,
             startstr=None,
             endstr=None,
             substrs=None,

lldb/test/CMakeLists.txt

set(LLDB_TEST_COMMON_ARGS
   -S nm
   -u CXXFLAGS
   -u CFLAGS
   )
 
 list(APPEND LLDB_TEST_COMMON_ARGS
   --executable ${LLDB_TEST_EXECUTABLE}
   --dsymutil ${LLDB_TEST_DSYMUTIL}
+  --filecheck ${LLDB_TEST_FILECHECK}
   -C ${LLDB_TEST_C_COMPILER}
   )
 
 if ( CMAKE_SYSTEM_NAME MATCHES "Windows" )
   # All tests are currently flaky on Windows, so rerun them all once when they fail.
   set(LLDB_TEST_COMMON_ARGS ${LLDB_TEST_COMMON_ARGS} --rerun-all-issues)
 
   set(LLDB_TEST_DEBUG_TEST_CRASHES