Fix TestGdbRemote* on remote Linux

[tberghammer]

Summary

Fix TestGdbRemote* on remote Linux

[vharron]

Is fixes the test but I'm getting these stack traces prints. What do they mean? (I'm in calibration now)

vharron@tifa:~/ll/svn/llvm/tools/lldb/test$ ./dotest.py -m --executable $HOME/ll/svn/build/host/bin/lldb -s $HOME/ll/svn/build/host/lldb-test-traces -A x86_64 -C clang-3.5 -u CFLAGS -u CXXFLAGS --platform-name remote-linux --platform-url connect://192.168.100.130:5432 --platform-working-dir /tmp -p TestLldbGdbServer.py
LLDB library dir: /usr/local/google/home/vharron/ll/svn/build/host/bin
lldb version 3.7.0 ( https://[e-mail]/svn/llvm-project/lldb/trunk revision 233281 clang revision 233281 llvm revision 233281)
Setting up remote platform 'remote-linux'
Connecting to remote platform 'remote-linux' at 'connect://192.168.100.130:5432'...
Connected.
Setting remote platform working directory to '/tmp'...
lldb.pre_flight: None
lldb.post_flight: None

Session logs for test failures/errors/unexpected successes will go into directory '/usr/local/google/home/vharron/ll/svn/build/host/lldb-test-traces'
Command invoked: ./dotest.py -m --executable /usr/local/google/home/vharron/ll/svn/build/host/bin/lldb -s /usr/local/google/home/vharron/ll/svn/build/host/lldb-test-traces -A x86_64 -C clang-3.5 -u CFLAGS -u CXXFLAGS --platform-name remote-linux --platform-url connect://192.168.100.130:5432 --platform-working-dir /tmp -p TestLldbGdbServer.py
compilers=['clang-3.5']

Configuration: arch=x86_64 compiler=clang-3.5

Collected 69 tests

.=success F=fail E=error s=skipped x=expected-fail u=unexpected-success

sus.s.s.s.s.swarning: output queue entries still exist:
\x00\x00

from here:

File "./dotest.py", line 1805, in <module>
  resultclass=LLDBTestResult).run(suite)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/runner.py", line 162, in run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 64, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 84, in run
  self._wrapped_run(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 114, in _wrapped_run
  test._wrapped_run(result, debug)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 116, in _wrapped_run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 415, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 357, in run
  testMethod()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 513, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 479, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 221, in test_c_packet_works_llgs_dwarf
  self.c_packet_works()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 207, in c_packet_works
  self.expect_gdbremote_sequence()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/gdbremote_testcase.py", line 517, in expect_gdbremote_sequence
  return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/lldbgdbserverutils.py", line 252, in expect_lldb_gdbserver_replay
  context["O_content"] = pump.get_accumulated_output()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/socket_packet_pump.py", line 69, in __exit__
  traceback.print_stack()

.s.s.swarning: output queue entries still exist:
\x00\x00

from here:

File "./dotest.py", line 1805, in <module>
  resultclass=LLDBTestResult).run(suite)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/runner.py", line 162, in run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 64, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 84, in run
  self._wrapped_run(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 114, in _wrapped_run
  test._wrapped_run(result, debug)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 116, in _wrapped_run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 415, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 357, in run
  testMethod()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 513, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 479, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 158, in test_inferior_exit_0_llgs_dwarf
  self.inferior_exit_0()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 144, in inferior_exit_0
  self.expect_gdbremote_sequence()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/gdbremote_testcase.py", line 517, in expect_gdbremote_sequence
  return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/lldbgdbserverutils.py", line 252, in expect_lldb_gdbserver_replay
  context["O_content"] = pump.get_accumulated_output()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/socket_packet_pump.py", line 69, in __exit__
  traceback.print_stack()

.swarning: output queue entries still exist:
\x00\x00

from here:

File "./dotest.py", line 1805, in <module>
  resultclass=LLDBTestResult).run(suite)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/runner.py", line 162, in run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 64, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 84, in run
  self._wrapped_run(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 114, in _wrapped_run
  test._wrapped_run(result, debug)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 116, in _wrapped_run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 415, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 357, in run
  testMethod()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 513, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 479, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 192, in test_inferior_exit_42_llgs_dwarf
  self.inferior_exit_42()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 178, in inferior_exit_42
  self.expect_gdbremote_sequence()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/gdbremote_testcase.py", line 517, in expect_gdbremote_sequence
  return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/lldbgdbserverutils.py", line 252, in expect_lldb_gdbserver_replay
  context["O_content"] = pump.get_accumulated_output()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/socket_packet_pump.py", line 69, in __exit__
  traceback.print_stack()

.swarning: output queue entries still exist:
hello, world\r\nhe

from here:

File "./dotest.py", line 1805, in <module>
  resultclass=LLDBTestResult).run(suite)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/runner.py", line 162, in run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 64, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 84, in run
  self._wrapped_run(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 114, in _wrapped_run
  test._wrapped_run(result, debug)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 116, in _wrapped_run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 415, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 357, in run
  testMethod()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 513, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 479, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 255, in test_inferior_print_exit_llgs_dwarf
  self.inferior_print_exit()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 240, in inferior_print_exit
  context = self.expect_gdbremote_sequence()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/gdbremote_testcase.py", line 517, in expect_gdbremote_sequence
  return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/lldbgdbserverutils.py", line 252, in expect_lldb_gdbserver_replay
  context["O_content"] = pump.get_accumulated_output()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/socket_packet_pump.py", line 69, in __exit__
  traceback.print_stack()

.s.s.s.s.s.s.s.s.s..s.s.s.s.s.s.s.s.s.s.s.s.swarning: output queue entries still exist:
message: Hello, memoryX\r\nme

from here:

File "./dotest.py", line 1805, in <module>
  resultclass=LLDBTestResult).run(suite)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/runner.py", line 162, in run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 64, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 84, in run
  self._wrapped_run(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 114, in _wrapped_run
  test._wrapped_run(result, debug)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/suite.py", line 116, in _wrapped_run
  test(result)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 415, in __call__
  return self.run(*args, **kwds)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/unittest2/case.py", line 357, in run
  testMethod()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 513, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/lldbtest.py", line 479, in wrapper
  return func(self, *args, **kwargs)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 1371, in test_written_M_content_reads_back_correctly_llgs_dwarf
  self.written_M_content_reads_back_correctly()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/TestLldbGdbServer.py", line 1349, in written_M_content_reads_back_correctly
  context = self.expect_gdbremote_sequence()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/gdbremote_testcase.py", line 517, in expect_gdbremote_sequence
  return expect_lldb_gdbserver_replay(self, self.sock, self.test_sequence, timeout_seconds, self.logger)
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/lldbgdbserverutils.py", line 252, in expect_lldb_gdbserver_replay
  context["O_content"] = pump.get_accumulated_output()
File "/usr/local/google/home/vharron/ll/svn/llvm/tools/lldb/test/tools/lldb-server/socket_packet_pump.py", line 69, in __exit__
  traceback.print_stack()

.

Ran 69 tests in 51.236s

OK (skipped=34, unexpected successes=1)
Session logs for test failures/errors/unexpected successes can be found in directory '/usr/local/google/home/vharron/ll/svn/build/host/lldb-test-traces'

test/tools/lldb-server/gdbremote_testcase.py

294 ↗

(On Diff #22283)

MAX_CONNECT_ATTEMPTS

[ted]

I'm seeing less failures, but I'm still seeing some with this patch:

connect to debug monitor on port 14060 failed, attempt #18 of 20
connect to debug monitor on port 12030 failed, attempt #19 of 20
connect to debug monitor on port 15778 failed, attempt #20 of 20
FAIL: LLDB (/pkg/qct/software/llvm/build_tools/llvm34_140306/bin/clang-x86_64) :: test_attach_commandline_qProcessInfo_reports_correct_pid_llgs_dwarf (TestGdbRemoteProcessInfo.TestGdbRemoteProcessInfo)

[tberghammer]

@vharron: I see the same warnings without this CL also when running the tests for local debugging so it isn't a new warning. I am not sure where they are coming from but it looks like some of the tests don't expect $O packets (or expects less) while lldb-server sends them. We should probably investigate it but I think it is out of the scope of this CL.

@ted: I tried to reproduce the failure but I only managed to do it when running dosep.py with multiple threads. Doing it is not fully supported (yet) because lldb-platform drops some connection when a lot of incoming connection came in the same time what makes the test suit flaky. If you can reproduce the failure with running dotest.py (on 1 thread) or with running dosep.py with LLDB_TEST_THREADS=1 set then please let me know (with some details about how you did it) and I will investigate it further.

[tberghammer]

Fix typo

[vharron]

Please submit at your early convenience!

lldb/trunk/test/dotest.py

if lldb_platform_name:
     print "Setting up remote platform '%s'" % (lldb_platform_name)
     lldb.remote_platform = lldb.SBPlatform(lldb_platform_name)
     if not lldb.remote_platform.IsValid():
         print "error: unable to create the LLDB platform named '%s'." % (lldb_platform_name)
         exitTestSuite(1)
     if lldb_platform_url:
         # We must connect to a remote platform if a LLDB platform URL was specified
         print "Connecting to remote platform '%s' at '%s'..." % (lldb_platform_name, lldb_platform_url)
-        platform_connect_options = lldb.SBPlatformConnectOptions(lldb_platform_url); 
+        lldb.platfrom_url = lldb_platform_url
+        platform_connect_options = lldb.SBPlatformConnectOptions(lldb_platform_url)
         err = lldb.remote_platform.ConnectRemote(platform_connect_options)
         if err.Success():
             print "Connected."
         else:
             print "error: failed to connect to remote platform using URL '%s': %s" % (lldb_platform_url, err)
             exitTestSuite(1)
-    
+    else:
+        lldb.platfrom_url = None
     if lldb_platform_working_dir:
         print "Setting remote platform working directory to '%s'..." % (lldb_platform_working_dir)
         lldb.remote_platform.SetWorkingDirectory(lldb_platform_working_dir)
     
     lldb.remote_platform_working_dir = lldb_platform_working_dir
     lldb.DBG.SetSelectedPlatform(lldb.remote_platform)
 else:
     lldb.remote_platform = None
     lldb.remote_platform_working_dir = None
+    lldb.platfrom_url = None
 
 target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
 
 # By default, both dsym and dwarf tests are performed.
 # Use @dsym_test or @dwarf_test decorators, defined in lldbtest.py, to mark a test
 # as a dsym or dwarf test.  Use '-N dsym' or '-N dwarf' to exclude dsym or dwarf
 # tests from running.
 dont_do_dsym_test = dont_do_dsym_test or "linux" in target_platform or "freebsd" in target_platform

lldb/trunk/test/tools/lldb-server/gdbremote_testcase.py

def setUp(self):
         self.logger.setLevel(self._LOGGING_LEVEL)
         self.test_sequence = GdbRemoteTestSequence(self.logger)
         self.set_inferior_startup_launch()
         self.port = self.get_next_port()
         self.named_pipe_path = None
         self.named_pipe = None
         self.named_pipe_fd = None
         self.stub_sends_two_stop_notifications_on_kill = False
+        if lldb.platfrom_url:
+            self.stub_hostname = re.match(".*://(.*):[0-9]+", lldb.platfrom_url).group(1)
+        else:
             self.stub_hostname = "localhost"
 
     def get_next_port(self):
         return 12000 + random.randint(0,3999)
 
     def reset_test_sequence(self):
         self.test_sequence = GdbRemoteTestSequence(self.logger)
 
     def create_named_pipe(self):
def forward_adb_port(self, source, target, direction):
         
         subprocess.call(["adb", direction, "tcp:%d" % source, "tcp:%d" % target])
         self.addTearDownHook(remove_port_forward)
 
     def create_socket(self):
         sock = socket.socket()
         logger = self.logger
 
+        triple = self.dbg.GetSelectedPlatform().GetTriple()
+        if re.match(".*-.*-.*-android", triple):
+            self.forward_adb_port(self.port, self.port, "forward")
+
+        connect_info = (self.stub_hostname, self.port)
+        sock.connect(connect_info)
+
         def shutdown_socket():
             if sock:
                 try:
                     # send the kill packet so lldb-server shuts down gracefully
                     sock.sendall(GdbRemoteTestCaseBase._GDBREMOTE_KILL_PACKET)
                 except:
                     logger.warning("failed to send kill packet to debug monitor: {}; ignoring".format(sys.exc_info()[0]))
 
                 try:
                     sock.close()
                 except:
                     logger.warning("failed to close socket to debug monitor: {}; ignoring".format(sys.exc_info()[0]))
 
         self.addTearDownHook(shutdown_socket)
 
-        triple = self.dbg.GetSelectedPlatform().GetTriple()
-        if re.match(".*-.*-.*-android", triple):
-            self.forward_adb_port(self.port, self.port, "forward")
-
-        connect_info = (self.stub_hostname, self.port)
-        # print "connecting to stub on {}:{}".format(connect_info[0], connect_info[1])
-        sock.connect(connect_info)
-
         return sock
 
     def set_inferior_startup_launch(self):
         self._inferior_startup = self._STARTUP_LAUNCH
 
     def set_inferior_startup_attach(self):
         self._inferior_startup = self._STARTUP_ATTACH
 
     def set_inferior_startup_attach_manually(self):
         self._inferior_startup = self._STARTUP_ATTACH_MANUALLY
 
     def get_debug_monitor_command_line_args(self, attach_pid=None):
+        if lldb.remote_platform:
+            commandline_args = self.debug_monitor_extra_args + ["*:{}".format(self.port)]
+        else:
             commandline_args = self.debug_monitor_extra_args + ["localhost:{}".format(self.port)]
+
         if attach_pid:
             commandline_args += ["--attach=%d" % attach_pid]
         if self.named_pipe_path:
             commandline_args += ["--named-pipe", self.named_pipe_path]
         return commandline_args
 
     def run_platform_command(self, cmd):
         platform = self.dbg.GetSelectedPlatform()
def connect_to_debug_monitor(self, attach_pid=None):
             logger = self.logger
             def shutdown_debug_monitor():
                 try:
                     server.terminate()
                 except:
                     logger.warning("failed to terminate server for debug monitor: {}; ignoring".format(sys.exc_info()[0]))
             self.addTearDownHook(shutdown_debug_monitor)
 
+            connect_attemps = 0
+            MAX_CONNECT_ATTEMPTS = 10
+
+            while connect_attemps < MAX_CONNECT_ATTEMPTS:
                 # Create a socket to talk to the server
                 try:
                     self.sock = self.create_socket()
                     return server
                 except socket.error as serr:
                     # We're only trying to handle connection refused.
                     if serr.errno != errno.ECONNREFUSED:
                         raise serr
-                # We should close the server here to be safe.
-                server.terminate()
+                time.sleep(0.5)
+                connect_attemps += 1
+
 
             # Increment attempts.
             print("connect to debug monitor on port %d failed, attempt #%d of %d" % (self.port, attempts + 1, MAX_ATTEMPTS))
             attempts += 1
 
             # And wait a random length of time before next attempt, to avoid collisions.
             time.sleep(random.randint(1,5))