[test] Correctly retry connections on android targets

[labath]

Summary

Normally, when the remote stub is not ready, we will get ECONNREFUSED during the connect()
attempt. However, due to the way how ADB forwarding works, on android targets the connect() will
always be successful, but the connection will be immediately dropped if ADB could not connect on
the remote side. This commit tries to detect this situation, and report it as "connection
refused" so that the upper test layers attempt the connection again.

[tberghammer]

Looks good

[labath]

Fix comment

lldb/trunk/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py

 import sys
 import tempfile
 import time
 from lldbsuite.test import configuration
 from lldbsuite.test.lldbtest import *
 from lldbgdbserverutils import *
 import logging
 
+class _ConnectionRefused(IOError):
+    pass
+
 class GdbRemoteTestCaseBase(TestBase):
 
     _TIMEOUT_SECONDS = 7
 
     _GDBREMOTE_KILL_PACKET = "$k#6b"
 
     # Start the inferior separately, attach to the inferior on the stub command line.
     _STARTUP_ATTACH = "attach"
class GdbRemoteTestCaseBase(TestBase):
     def forward_adb_port(self, source, target, direction, device):
         adb = [ 'adb' ] + ([ '-s', device ] if device else []) + [ direction ]
         def remove_port_forward():
             subprocess.call(adb + [ "--remove", "tcp:%d" % source])
 
         subprocess.call(adb + [ "tcp:%d" % source, "tcp:%d" % target])
         self.addTearDownHook(remove_port_forward)
 
+    def _verify_socket(self, sock):
+        # Normally, when the remote stub is not ready, we will get ECONNREFUSED during the
+        # connect() attempt. However, due to the way how ADB forwarding works, on android targets
+        # the connect() will always be successful, but the connection will be immediately dropped
+        # if ADB could not connect on the remote side. This function tries to detect this
+        # situation, and report it as "connection refused" so that the upper layers attempt the
+        # connection again.
+        triple = self.dbg.GetSelectedPlatform().GetTriple()
+        if not re.match(".*-.*-.*-android", triple):
+            return # Not android.
+        can_read, _, _ = select.select([sock], [], [], 0.1)
+        if sock not in can_read:
+            return # Data is not available, but the connection is alive.
+        if len(sock.recv(1, socket.MSG_PEEK)) == 0:
+            raise _ConnectionRefused() # Got EOF, connection dropped.
+
     def create_socket(self):
         sock = socket.socket()
         logger = self.logger
 
         triple = self.dbg.GetSelectedPlatform().GetTriple()
         if re.match(".*-.*-.*-android", triple):
             self.forward_adb_port(self.port, self.port, "forward", self.stub_device)
 
         logger.info("Connecting to debug monitor on %s:%d", self.stub_hostname, self.port)
         connect_info = (self.stub_hostname, self.port)
+        try:
             sock.connect(connect_info)
+        except socket.error as serr:
+            if serr.errno == errno.ECONNREFUSED:
+                raise _ConnectionRefused()
+            raise serr
 
         def shutdown_socket():
             if sock:
                 try:
                     # send the kill packet so lldb-server shuts down gracefully
                     sock.sendall(GdbRemoteTestCaseBase._GDBREMOTE_KILL_PACKET)
                 except:
                     logger.warning("failed to send kill packet to debug monitor: {}; ignoring".format(sys.exc_info()[0]))
 
                 try:
                     sock.close()
                 except:
                     logger.warning("failed to close socket to debug monitor: {}; ignoring".format(sys.exc_info()[0]))
 
         self.addTearDownHook(shutdown_socket)
 
+        self._verify_socket(sock)
+
         return sock
 
     def set_inferior_startup_launch(self):
         self._inferior_startup = self._STARTUP_LAUNCH
 
     def set_inferior_startup_attach(self):
         self._inferior_startup = self._STARTUP_ATTACH
 
def connect_to_debug_monitor(self, attach_pid=None):
             self.addTearDownHook(shutdown_debug_monitor)
 
             connect_attemps = 0
             MAX_CONNECT_ATTEMPTS = 10
 
             while connect_attemps < MAX_CONNECT_ATTEMPTS:
                 # Create a socket to talk to the server
                 try:
+                    logger.info("Connect attempt %d", connect_attemps+1)
                     self.sock = self.create_socket()
                     return server
-                except socket.error as serr:
-                    # We're only trying to handle connection refused.
-                    if serr.errno != errno.ECONNREFUSED:
-                        raise serr
+                except _ConnectionRefused as serr:
+                    # Ignore, and try again.
+                    pass
                 time.sleep(0.5)
                 connect_attemps += 1
 
             # We should close the server here to be safe.
             server.terminate()
 
             # Increment attempts.
             print("connect to debug monitor on port %d failed, attempt #%d of %d" % (self.port, attempts + 1, MAX_ATTEMPTS))