dotest: improvements to the pexpect tests

[labath]

Summary

While working on r370054, i've found it frustrating that the test output
was compeletely unhelpful in case of failures. Therefore I've decided to
improve that. In this I reuse the PExpectTest class, which was one of
our mechanisms for running pexpect tests, but which has gotten orhpaned
in the mean time.

I've replaced the existing send methods with a "expect" method, which
I've tried to design so that it has a similar interface to the expect
method in regular non-pexpect dotest tests (as it essentially does
something very similar). I've kept the ability to dump the transcript of
the pexpect communication to stdout in the "trace" mode, as that is a
very handy way to figure out what the test is doing. I've also removed
the "expect_string" method used in the existing tests -- I've found this
to be unhelpful because it hides the message that would be normally
displayed by the EOF exception. Although vebose, this message includes
some important information, like what strings we were searching for,
what were the last bits of lldb output, etc. I've also beefed up the
class to automatically disable the debug info test duplication, and
auto-skip tests when the host platform does not support pexpect.

This patch ports TestMultilineCompletion and TestIOHandlerCompletion to
the new class. It also deletes TestFormats as it is not testing anything
(definitely not formats) -- it used to test something at one point, but
then the testing code was commented out, and later deleted in (r356000).

[teemperor]

LGTM. And it seems like you found TestFormats.py at the same time as me and ninja'd D66962

lldb/trunk/packages/Python/lldbsuite/test/expression_command/multiline-completion/TestMultilineCompletion.py

 """
 Test completion for multiline expressions.
 """
 
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
+from lldbsuite.test.lldbpexpect import PExpectTest
 
-class MultilineCompletionTest(TestBase):
+class MultilineCompletionTest(PExpectTest):
 
     mydir = TestBase.compute_mydir(__file__)
-    NO_DEBUG_INFO_TESTCASE = True
 
-    def setUp(self):
-        TestBase.setUp(self)
-        self.source = 'main.c'
-
-    def expect_string(self, string):
-        import pexpect
-        """This expects for "string", with timeout & EOF being test fails."""
-        try:
-            self.child.expect_exact(string)
-        except pexpect.EOF:
-            self.fail("Got EOF waiting for '%s'" % (string))
-        except pexpect.TIMEOUT:
-            self.fail("Timed out waiting for '%s'" % (string))
-
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows")
     def test_basic_completion(self):
         """Test that we can complete a simple multiline expression"""
         self.build()
-        self.setTearDownCleanup()
 
-        import pexpect
-        exe = self.getBuildArtifact("a.out")
         prompt = "(lldb) "
 
-        run_commands = ' -o "b main" -o "r"'
-        self.child = pexpect.spawn(
-            '%s %s %s %s' %
-            (lldbtest_config.lldbExec, self.lldbOption, run_commands, exe))
-        child = self.child
+        self.launch(executable=self.getBuildArtifact("a.out"))
+        self.expect("b main", substrs=["Breakpoint 1", "address ="])
+        self.expect("run", substrs=["stop reason ="])
 
-        self.expect_string(prompt)
         self.child.sendline("expr")
-        self.expect_string("terminate with an empty line to evaluate")
+        self.child.expect_exact("terminate with an empty line to evaluate")
         self.child.send("to_\t")
-        self.expect_string("to_complete")
+        self.child.expect_exact("to_complete")
 
-        self.deletePexpectChild()
+        self.child.send("\n\n")
+        self.expect_prompt()
+
+        self.quit()

lldb/trunk/packages/Python/lldbsuite/test/functionalities/format/Makefile

-LEVEL = ../../make
-
-C_SOURCES := main.c
-
-include $(LEVEL)/Makefile.rules

lldb/trunk/packages/Python/lldbsuite/test/functionalities/format/TestFormats.py

-"""
-Test the command history mechanism
-"""
-
-from __future__ import print_function
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class TestFormats(TestBase):
-
-    mydir = TestBase.compute_mydir(__file__)
-
-    @expectedFailureAll(
-        hostoslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows")
-    def test_formats(self):
-        """Test format string functionality."""
-        self.build()
-        exe = self.getBuildArtifact("a.out")
-        import pexpect
-        prompt = "(lldb) "
-        child = pexpect.spawn(
-            '%s %s -x -o "b main" -o r %s' %
-            (lldbtest_config.lldbExec, self.lldbOption, exe))
-        # So that the spawned lldb session gets shutdown durng teardown.
-        self.child = child
-
-        # Substitute 'Help!' for 'help' using the 'commands regex' mechanism.
-        child.expect_exact(prompt + 'target create "%s"' % exe)
-        child.expect_exact(prompt + 'b main')
-        child.expect_exact(prompt + 'r')
-        child.expect_exact(prompt)
-        child.sendline()

lldb/trunk/packages/Python/lldbsuite/test/functionalities/format/main.c

-//===-- main.c --------------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <stdio.h>
-int main (int argc, char const *argv[])
-{
-    printf("testing\n");
-    return 0;
-}

lldb/trunk/packages/Python/lldbsuite/test/iohandler/completion/TestIOHandlerCompletion.py

 """
 Test completion in our IOHandlers.
 """
 
 import os
 
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
+from lldbsuite.test.lldbpexpect import PExpectTest
 
-class IOHandlerCompletionTest(TestBase):
+class IOHandlerCompletionTest(PExpectTest):
 
     mydir = TestBase.compute_mydir(__file__)
-    NO_DEBUG_INFO_TESTCASE = True
 
-    def setUp(self):
-        TestBase.setUp(self)
-
-    def expect_string(self, string):
-        import pexpect
-        """This expects for "string", with timeout & EOF being test fails."""
-        try:
-            self.child.expect_exact(string)
-        except pexpect.EOF:
-            self.fail("Got EOF waiting for '%s'" % (string))
-        except pexpect.TIMEOUT:
-            self.fail("Timed out waiting for '%s'" % (string))
-
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows")
     def test_completion(self):
-        self.setTearDownCleanup()
-
-        import pexpect
-        exe = self.getBuildArtifact("a.out")
-        prompt = "(lldb) "
-
-        self.child = pexpect.spawn(
-            '%s %s %s %s' %
-            (lldbtest_config.lldbExec, self.lldbOption, "", exe),
-            dimensions=(100, 500))
+        self.launch(dimensions=(100,500))
 
-        self.expect_string(prompt)
         # Start tab completion, go to the next page and then display all with 'a'.
         self.child.send("\t\ta")
-        self.expect_string("register")
+        self.child.expect_exact("register")
 
         # Try tab completing regi to register.
         self.child.send("regi\t")
-        self.expect_string(prompt + "register")
+        self.child.expect_exact(self.PROMPT + "register")
         self.child.send("\n")
+        self.expect_prompt()
 
         # Try tab completing directories and files. Also tests the partial
         # completion where LLDB shouldn't print a space after the directory
         # completion (as it didn't completed the full token).
         dir_without_slashes = os.path.realpath(os.path.dirname(__file__)).rstrip("/")
         self.child.send("file " + dir_without_slashes + "\t")
-        self.expect_string("iohandler/completion/")
+        self.child.expect_exact("iohandler/completion/")
         # If we get a correct partial completion without a trailing space, then this
         # should complete the current test file.
         self.child.send("TestIOHandler\t")
-        self.expect_string("TestIOHandlerCompletion.py")
+        self.child.expect_exact("TestIOHandlerCompletion.py")
         self.child.send("\n")
+        self.expect_prompt()
 
         # Start tab completion and abort showing more commands with 'n'.
         self.child.send("\t")
-        self.expect_string("More (Y/n/a)")
+        self.child.expect_exact("More (Y/n/a)")
         self.child.send("n")
-        self.expect_string(prompt)
+        self.expect_prompt()
 
         # Shouldn't crash or anything like that.
         self.child.send("regoinvalid\t")
-        self.expect_string(prompt)
+        self.expect_prompt()
 
-        self.deletePexpectChild()
+        self.quit()

lldb/trunk/packages/Python/lldbsuite/test/lldbpexpect.py

 from __future__ import print_function
 from __future__ import absolute_import
 
 # System modules
 import sys
 
 # Third-party modules
 import six
 
 # LLDB Modules
 import lldb
 from .lldbtest import *
 from . import lldbutil
 
 if sys.platform.startswith('win32'):
-    class PExpectTest(TestBase):
+    # llvm.org/pr22274: need a pexpect replacement for windows
+    class PExpectTest(object):
         pass
 else:
     import pexpect
 
     class PExpectTest(TestBase):
 
-        mydir = TestBase.compute_mydir(__file__)
-
-        def setUp(self):
-            TestBase.setUp(self)
+        NO_DEBUG_INFO_TESTCASE = True
+        PROMPT = "(lldb) "
 
-        def launchArgs(self):
-            return ""
+        def expect_prompt(self):
+            self.child.expect_exact(self.PROMPT)
 
-        def launch(self, timeout=None):
-            if timeout is None:
-                timeout = 30
+        def launch(self, executable=None, timeout=30, dimensions=None):
             logfile = sys.stdout if self.TraceOn() else None
+            args = ['--no-lldbinit', '--no-use-colors']
+            for cmd in self.setUpCommands():
+                args += ['-O', cmd]
+            if executable is not None:
+                args += ['--file', executable]
             self.child = pexpect.spawn(
-                '%s --no-use-colors %s' %
-                (lldbtest_config.lldbExec, self.launchArgs()), logfile=logfile)
-            self.child.timeout = timeout
-            self.timeout = timeout
-
-        def expect(self, patterns=None, timeout=None, exact=None):
-            if patterns is None:
-                return None
-            if timeout is None:
-                timeout = self.timeout
-            if exact is None:
-                exact = False
-            if exact:
-                return self.child.expect_exact(patterns, timeout=timeout)
-            else:
-                return self.child.expect(patterns, timeout=timeout)
+                    lldbtest_config.lldbExec, args=args, logfile=logfile,
+                    timeout=timeout, dimensions=dimensions)
+            self.expect_prompt()
+            for cmd in self.setUpCommands():
+                self.child.expect_exact(cmd)
+                self.expect_prompt()
+            if executable is not None:
+                self.child.expect_exact("target create")
+                self.child.expect_exact("Current executable set to")
+                self.expect_prompt()
+
+        def expect(self, cmd, substrs=None):
+            self.child.sendline(cmd)
+            if substrs is not None:
+                for s in substrs:
+                    self.child.expect_exact(s)
+            self.expect_prompt()
 
-        def expectall(self, patterns=None, timeout=None, exact=None):
-            if patterns is None:
-                return None
-            if timeout is None:
-                timeout = self.timeout
-            if exact is None:
-                exact = False
-            for pattern in patterns:
-                self.expect(pattern, timeout=timeout, exact=exact)
-
-        def sendimpl(
-                self,
-                sender,
-                command,
-                patterns=None,
-                timeout=None,
-                exact=None):
-            sender(command)
-            return self.expect(patterns=patterns, timeout=timeout, exact=exact)
-
-        def send(self, command, patterns=None, timeout=None, exact=None):
-            return self.sendimpl(
-                self.child.send,
-                command,
-                patterns,
-                timeout,
-                exact)
-
-        def sendline(self, command, patterns=None, timeout=None, exact=None):
-            return self.sendimpl(
-                self.child.sendline,
-                command,
-                patterns,
-                timeout,
-                exact)
-
-        def quit(self, gracefully=None):
-            if gracefully is None:
-                gracefully = True
+        def quit(self, gracefully=True):
             self.child.sendeof()
             self.child.close(force=not gracefully)
             self.child = None