[test] Persist packets between expect_gdbremote_sequence invocations

[labath]

Summary

Some tests (Hc_then_Csignal_signals_correct_thread, at least) were sending a "continue" packet in
one expect_gdbremote_sequence invocation, and "expecting" the stop-reply in another call. This
posed a problem, because the were packets were not persisted between the two invocations, and if
the stub was exceptionally fast to respond, the packet would be received in the first invocation
(where it would be ignored) and then the second invocation would fail because it could not find
the packet.

Since doing matching in two invocations seems like a reasonable use of the packet pump, instead
of fixing the test, I make sure the packet_pump supports this usage by making the list of
captured packets persistent.

[tfiala]

LGTM, Pavel!

lldb/trunk/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py

def isVerboseLoggingRequested(self):
         return any(("gdb-remote" in channel) for channel in lldbtest_config.channels)
 
     def setUp(self):
         TestBase.setUp(self)
 
         self.setUpBaseLogging()
         self._remote_server_log_file = None
         self.debug_monitor_extra_args = []
+        self._pump_queues = socket_packet_pump.PumpQueues()
 
         if self.isVerboseLoggingRequested():
             # If requested, full logs go to a log file
             self._verbose_log_handler = logging.FileHandler(self.log_basename + "-host.log")
             self._verbose_log_handler.setFormatter(self._log_formatter)
             self._verbose_log_handler.setLevel(logging.DEBUG)
             self.logger.addHandler(self._verbose_log_handler)
 
def setUp(self):
                 self.stub_hostname = 'localhost'
             else:
                 self.stub_device = None
                 self.stub_hostname = host
         else:
             self.stub_hostname = "localhost"
 
     def tearDown(self):
+        self._pump_queues.verify_queues_empty()
+
         if self._remote_server_log_file is not None:
             lldb.remote_platform.Get(lldb.SBFileSpec(self._remote_server_log_file),
                     lldb.SBFileSpec(self.getLocalServerLogFile()))
             lldb.remote_platform.Run(lldb.SBPlatformShellCommand("rm " + self._remote_server_log_file))
             self._remote_server_log_file = None
 
         self.logger.removeHandler(self._verbose_log_handler)
         self._verbose_log_handler = None
def parse_register_info_packets(self, context):
         self.assertIsNotNone(reg_info_responses)
 
         # Parse register infos.
         return [parse_reg_info_response(reg_info_response) for reg_info_response in reg_info_responses]
 
     def expect_gdbremote_sequence(self, timeout_seconds=None):
         if not timeout_seconds:
             timeout_seconds = self._TIMEOUT_SECONDS
-        return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
+        return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence,
+                self._pump_queues, timeout_seconds, self.logger)
 
     _KNOWN_REGINFO_KEYS = [
         "name",
         "alt-name",
         "bitsize",
         "offset",
         "encoding",
         "format",

lldb/trunk/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py

def assert_packets_equal(asserter, actual_packet, expected_packet):
     actual_stripped = _STRIP_CHECKSUM_REGEX.sub('', actual_packet)
     expected_stripped = _STRIP_CHECKSUM_REGEX.sub('', expected_packet)
     asserter.assertEqual(actual_stripped, expected_stripped)
 
 def expect_lldb_gdbserver_replay(
     asserter,
     sock,
     test_sequence,
+    pump_queues,
     timeout_seconds,
     logger=None):
     """Replay socket communication with lldb-gdbserver and verify responses.
 
     Args:
         asserter: the object providing assertEqual(first, second, msg=None), e.g. TestCase instance.
 
         sock: the TCP socket connected to the lldb-gdbserver exe.
Returns:
         O packets received.
     """
     
     # Ensure we have some work to do.
     if len(test_sequence.entries) < 1:
         return {}
 
     context = {"O_count":0, "O_content":""}
-    with socket_packet_pump.SocketPacketPump(sock, logger) as pump:
+    with socket_packet_pump.SocketPacketPump(sock, pump_queues, logger) as pump:
         # Grab the first sequence entry.
         sequence_entry = test_sequence.entries.pop(0)
         
         # While we have an active sequence entry, send messages
         # destined for the stub and collect/match/process responses
         # expected from the stub.
         while sequence_entry:
             if sequence_entry.is_send_to_remote():
with socket_packet_pump.SocketPacketPump(sock, pump_queues, logger) as pump:
                 # This is an entry expecting to receive content from the remote debug monitor.
 
                 # We'll pull from (and wait on) the queue appropriate for the type of matcher.
                 # We keep separate queues for process output (coming from non-deterministic
                 # $O packet division) and for all other packets.
                 if sequence_entry.is_output_matcher():
                     try:
                         # Grab next entry from the output queue.
-                        content = pump.output_queue().get(True, timeout_seconds)
+                        content = pump_queues.output_queue().get(True, timeout_seconds)
                     except queue.Empty:
                         if logger:
                             logger.warning("timeout waiting for stub output (accumulated output:{})".format(pump.get_accumulated_output()))
                         raise Exception("timed out while waiting for output match (accumulated output: {})".format(pump.get_accumulated_output()))
                 else:
                     try:
-                        content = pump.packet_queue().get(True, timeout_seconds)
+                        content = pump_queues.packet_queue().get(True, timeout_seconds)
                     except queue.Empty:
                         if logger:
                             logger.warning("timeout waiting for packet match (receive buffer: {})".format(pump.get_receive_buffer()))
                         raise Exception("timed out while waiting for packet match (receive buffer: {})".format(pump.get_receive_buffer()))
                 
                 # Give the sequence entry the opportunity to match the content.
                 # Output matchers might match or pass after more output accumulates.
                 # Other packet types generally must match.

lldb/trunk/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py

def _handle_output_packet_string(packet_contents):
     else:
         return packet_contents[1:].decode("hex")
 
 def _dump_queue(the_queue):
     while not the_queue.empty():
         print(codecs.encode(the_queue.get(True), "string_escape"))
         print("\n")
 
+class PumpQueues(object):
+    def __init__(self):
+        self._output_queue = queue.Queue()
+        self._packet_queue = queue.Queue()
+
+    def output_queue(self):
+        return self._output_queue
+
+    def packet_queue(self):
+        return self._packet_queue
+
+    def verify_queues_empty(self):
+        # Warn if there is any content left in any of the queues.
+        # That would represent unmatched packets.
+        if not self.output_queue().empty():
+            print("warning: output queue entries still exist:")
+            _dump_queue(self.output_queue())
+            print("from here:")
+            traceback.print_stack()
+
+        if not self.packet_queue().empty():
+            print("warning: packet queue entries still exist:")
+            _dump_queue(self.packet_queue())
+            print("from here:")
+            traceback.print_stack()
+
+
 class SocketPacketPump(object):
     """A threaded packet reader that partitions packets into two streams.
 
     All incoming $O packet content is accumulated with the current accumulation
     state put into the OutputQueue.
 
     All other incoming packets are placed in the packet queue.
 
     A select thread can be started and stopped, and runs to place packet
     content into the two queues.
     """
 
     _GDB_REMOTE_PACKET_REGEX = re.compile(r'^\$([^\#]*)#[0-9a-fA-F]{2}')
 
-    def __init__(self, pump_socket, logger=None):
+    def __init__(self, pump_socket, pump_queues, logger=None):
         if not pump_socket:
             raise Exception("pump_socket cannot be None")
 
-        self._output_queue = queue.Queue()
-        self._packet_queue = queue.Queue()
         self._thread = None
         self._stop_thread = False
         self._socket = pump_socket
         self._logger = logger
         self._receive_buffer = ""
         self._accumulated_output = ""
+        self._pump_queues = pump_queues
 
     def __enter__(self):
         """Support the python 'with' statement.
 
         Start the pump thread."""
         self.start_pump_thread()
         return self
 
     def __exit__(self, exit_type, value, the_traceback):
         """Support the python 'with' statement.
 
         Shut down the pump thread."""
         self.stop_pump_thread()
 
-        # Warn if there is any content left in any of the queues.
-        # That would represent unmatched packets.
-        if not self.output_queue().empty():
-            print("warning: output queue entries still exist:")
-            _dump_queue(self.output_queue())
-            print("from here:")
-            traceback.print_stack()
-
-        if not self.packet_queue().empty():
-            print("warning: packet queue entries still exist:")
-            _dump_queue(self.packet_queue())
-            print("from here:")
-            traceback.print_stack()
-
     def start_pump_thread(self):
         if self._thread:
             raise Exception("pump thread is already running")
         self._stop_thread = False
         self._thread = threading.Thread(target=self._run_method)
         self._thread.start()
 
     def stop_pump_thread(self):
         self._stop_thread = True
         if self._thread:
             self._thread.join()
 
-    def output_queue(self):
-        return self._output_queue
-
-    def packet_queue(self):
-        return self._packet_queue
-
     def _process_new_bytes(self, new_bytes):
         if not new_bytes:
             return
         if len(new_bytes) < 1:
             return
 
         # Add new bytes to our accumulated unprocessed packet bytes.
         self._receive_buffer += new_bytes
 
         # Parse fully-formed packets into individual packets.
         has_more = len(self._receive_buffer) > 0
         while has_more:
             if len(self._receive_buffer) <= 0:
                 has_more = False
             # handle '+' ack
             elif self._receive_buffer[0] == "+":
-                self._packet_queue.put("+")
+                self._pump_queues.packet_queue().put("+")
                 self._receive_buffer = self._receive_buffer[1:]
                 if self._logger:
                     self._logger.debug(
                         "parsed packet from stub: +\n" +
                         "new receive_buffer: {}".format(
                             self._receive_buffer))
             else:
                 packet_match = self._GDB_REMOTE_PACKET_REGEX.match(
                     self._receive_buffer)
                 if packet_match:
                     # Our receive buffer matches a packet at the
                     # start of the receive buffer.
                     new_output_content = _handle_output_packet_string(
                         packet_match.group(1))
                     if new_output_content:
                         # This was an $O packet with new content.
                         self._accumulated_output += new_output_content
-                        self._output_queue.put(self._accumulated_output)
+                        self._pump_queues.output_queue().put(self._accumulated_output)
                     else:
                         # Any packet other than $O.
-                        self._packet_queue.put(packet_match.group(0))
+                        self._pump_queues.packet_queue().put(packet_match.group(0))
 
                     # Remove the parsed packet from the receive
                     # buffer.
                     self._receive_buffer = self._receive_buffer[
                         len(packet_match.group(0)):]
                     if self._logger:
                         self._logger.debug(
                             "parsed packet from stub: " +