[dotest] Support specifying a version for skipIfOutOfTreeDebugserver

[JDevlieghere]

Summary

The skipIfOutOfTreeDebugserver decorator is meant to skip tests that exercise functionalities that may not be present yet in a debugserver that ships with Xcode. This is inherently tied to a version of debugserver and therefore a version should be part of the decorator. This patch extends the decorator to take an operator and a debugserver version.

This patch also removes the --out-of-tree-debugserver dotest flag. Instead we expect --server to be passed even when using a system debugserver, which is used to deduct the current version.

[JDevlieghere]

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py

18

I used 9999 here because that's commonly the placeholder until the next release.

[labath]

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py

18

I haven't looked at the code in detail, but given that you are removing the explicit notion of an "out of tree" debug server, it seems to be that this will kick in even if you are testing against the in-tree debug server, which doesn't seem right. Or am I missing something?

[aprantl]

lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/TestRegisters.py

73

This will not skip the test for an in-tree llvm.org debugserver, right?

73

(with a version number of 10)

[JDevlieghere]

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py

18

Yeah, I was just going to check for version numbers below a hundred, but then forgot to add it before I uploaded the patch.

[aprantl]

lldb/packages/Python/lldbsuite/test/decorators.py

513

IMHO the expression if-else is only good for readability if the expression fits on one line :-)

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py

18

Did you file a radar to update these tests once the version number is known?

[JDevlieghere]

This is blocked on debugserver not flushing its output when not running under a TTY. I added a real -version flag to debugserver that writes to stdout directory (r373127) but that's not very helpful when we're trying to detect older versions.

[JDevlieghere]

Abandoning this for the reason mentioned above.

lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/TestRegisters.py

def test_fp_register_write(self):
         self.build()
         self.fp_register_write()
 
     @skipIfiOSSimulator
     # "register read fstat" always return 0xffff
     @expectedFailureAndroid(archs=["i386"])
     @skipIfFreeBSD  # llvm.org/pr25057
     @skipIf(archs=no_match(['amd64', 'i386', 'x86_64']))
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<', '1100'])
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr37995")
     @expectedFailureNetBSD
     def test_fp_special_purpose_register_read(self):
         """Test commands that read fpu special purpose registers."""
         self.build()
         self.fp_special_purpose_register_read()
 
     @skipIfiOSSimulator

lldb/packages/Python/lldbsuite/test/configuration.py

 # Categories we want to skip
 skipCategories = ["darwin-log"]
 # use this to track per-category failures
 failuresPerCategory = {}
 
 # The path to LLDB.framework is optional.
 lldbFrameworkPath = None
 
+# The path to debugserver is optional.
+debugserver = None
+
 # Test suite repeat count.  Can be overwritten with '-# count'.
 count = 1
 
 # The 'arch' and 'compiler' can be specified via command line.
 arch = None        # Must be initialized after option parsing
 compiler = None    # Must be initialized after option parsing
 
 # The overriden dwarf verison.

lldb/packages/Python/lldbsuite/test/decorators.py

 def expectedFlakeyAndroid(bugnumber=None, api_levels=None, archs=None):
     return expectedFlakey(
         _skip_for_android(
             "flakey on android",
             api_levels,
             archs),
         bugnumber)
 
-def skipIfOutOfTreeDebugserver(func):
+def skipIfOutOfTreeDebugserver(debugserver_version=None):
     """Decorate the item to skip tests if using an out-of-tree debugserver."""
-    def is_out_of_tree_debugserver():
-        return "out-of-tree debugserver" if lldbtest_config.out_of_tree_debugserver else None
-    return skipTestIfFn(is_out_of_tree_debugserver)(func)
+    def is_out_of_tree_debugserver(self):
+        return "out-of-tree debugserver" if _check_expected_version(
+            debugserver_version[0],
+            debugserver_version[1],
+            self.getDebugserverVersion()) else None
+    return skipTestIfFn(is_out_of_tree_debugserver)
 
 def skipIfRemote(func):
     """Decorate the item to skip tests if testing remotely."""
     def is_remote():
         return "skip on remote platform" if lldb.remote_platform else None
     return skipTestIfFn(is_remote)(func)
 
 
 def skipIfNoSBHeaders(func):
     """Decorate the item to mark tests that should be skipped when LLDB is built with no SB API headers."""

lldb/packages/Python/lldbsuite/test/dotest.py

if not configuration.get_filecheck_path():
         logging.warning('(Double-check the --filecheck argument to dotest.py)')
 
     if args.channels:
         lldbtest_config.channels = args.channels
 
     if args.log_success:
         lldbtest_config.log_success = args.log_success
 
-    if args.out_of_tree_debugserver:
-        lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver
-
     # Set SDKROOT if we are using an Apple SDK
     if platform_system == 'Darwin' and args.apple_sdk:
         os.environ['SDKROOT'] = seven.get_command_output(
             'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' %
             (args.apple_sdk))
 
     if args.arch:
         configuration.arch = args.arch
if args.executable:
             lldbtest_config.lldbExec = which(args.executable)
         if not is_exe(lldbtest_config.lldbExec):
             logging.error(
                     '%s is not a valid executable to test; aborting...',
                     args.executable)
             sys.exit(-1)
 
     if args.server:
+        configuration.debugserver = args.server
         os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
 
     if args.excluded:
         for excl_file in args.excluded:
             parseExclusion(excl_file)
 
     if args.p:
         if args.p.startswith('-'):

lldb/packages/Python/lldbsuite/test/dotest_args.py

group.add_argument(
         '--executable',
         metavar='executable-path',
         help='The path to the lldb executable')
     group.add_argument(
         '--server',
         metavar='server-path',
         help='The path to the debug server executable to use')
     group.add_argument(
-        '--out-of-tree-debugserver',
-        dest='out_of_tree_debugserver',
-        action='store_true',
-        help='A flag to indicate an out-of-tree debug server is being used')
-    group.add_argument(
         '--dwarf-version',
         metavar='dwarf_version',
         dest='dwarf_version',
         type=int,
         help='Override the DWARF version.')
     group.add_argument(
         '-s',
         metavar='name',

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py

 @skipIfWindows
 class ConcurrentManyBreakpoints(ConcurrentEventsBase):
 
     mydir = ConcurrentEventsBase.compute_mydir(__file__)
 
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<=', '9999'])
     def test(self):
         """Test 100 breakpoints from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
         self.do_thread_actions(num_breakpoint_threads=100)

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py

 @skipIfWindows
 class ConcurrentManyCrash(ConcurrentEventsBase):
 
     mydir = ConcurrentEventsBase.compute_mydir(__file__)
 
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<=', '9999'])
     def test(self):
         """Test 100 threads that cause a segfault."""
         self.build(dictionary=self.getBuildFlags())
         self.do_thread_actions(num_crash_threads=100)

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py

class ConcurrentManySignals(ConcurrentEventsBase):
 
     mydir = ConcurrentEventsBase.compute_mydir(__file__)
 
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     # This test is flaky on Darwin.
     @skipIfDarwin
     @expectedFailureNetBSD
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<=', '9999'])
     def test(self):
         """Test 100 signals from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
         self.do_thread_actions(num_signal_threads=100)

lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py

 class ConcurrentManyWatchpoints(ConcurrentEventsBase):
 
     mydir = ConcurrentEventsBase.compute_mydir(__file__)
 
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
     @add_test_categories(["watchpoint"])
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<=', '9999'])
     def test(self):
         """Test 100 watchpoints from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
         self.do_thread_actions(num_watchpoint_threads=100)

lldb/packages/Python/lldbsuite/test/lldbtest.py

def getCompilerVersion(self):
         compiler = self.getCompilerBinary()
         version_output = system([[compiler, "-v"]])[1]
         for line in version_output.split(os.linesep):
             m = re.search('version ([0-9\.]+)', line)
             if m:
                 version = m.group(1)
         return version
 
+    def getDebugserverVersion(self):
+        """ Returns a string that represents the debugserver version. """
+        if not configuration.debugserver:
+            return '0'
+        try:
+            version_output = check_output([configuration.debugserver, "-v"], stderr=STDOUT)
+        except CalledProcessError as e:
+            version_output = e.output
+        for line in version_output.split(os.linesep):
+            m = re.search('PROJECT:lldb-([0-9\.]+)', line)
+            if m:
+                return m.group(1)
+        return '0'
+
     def getDwarfVersion(self):
         """ Returns the dwarf version generated by clang or '0'. """
         if configuration.dwarf_version:
             return str(configuration.dwarf_version)
         if 'clang' in self.getCompiler():
             try:
                 driver_output = check_output(
                     [self.getCompiler()] + '-g -c -x c - -o - -###'.split(),
def expectedCompilerVersion(self, compiler_version):
         if (operator == '<'):
             return LooseVersion(self.getCompilerVersion()) < LooseVersion(version)
         if (operator == '<=' or operator == '=<'):
             return LooseVersion(self.getCompilerVersion()) <= LooseVersion(version)
         if (operator == '!=' or operator == '!' or operator == 'not'):
             return str(version) not in str(self.getCompilerVersion())
         return str(version) in str(self.getCompilerVersion())
 
+    def expectedDebugserverVersion(self, debugserver_version):
+        if (debugserver_version is None):
+            return True
+        operator = str(debugserver_version[0])
+        version = str(compiler_version[1])
+
+
     def expectedCompiler(self, compilers):
         """Returns True iff any element of compilers is a sub-string of the current compiler."""
         if (compilers is None):
             return True
 
         for compiler in compilers:
             if compiler in self.getCompiler():
                 return True

lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py

def run_test_g_packet(self):
         self.test_sequence.add_log_lines(
             ["read packet: $G" + register_bank + "#00",
              {"direction": "send", "regex": r"^\$(.+)#[0-9a-fA-F]{2}$",
               "capture": {1: "G_reply"}}],
             True)
         context = self.expect_gdbremote_sequence()
         self.assertTrue(context.get("G_reply")[0] != 'E')
 
-    @skipIfOutOfTreeDebugserver
+    @skipIfOutOfTreeDebugserver(['<', '1100'])
     @debugserver_test
     @skipIfDarwinEmbedded
     def test_g_packet_debugserver(self):
         self.init_debugserver_test()
         self.run_test_g_packet()
 
     @skipIf(archs=no_match(["x86_64"]))
     def g_returns_correct_data(self, with_suffix):

lldb/test/CMakeLists.txt

add_custom_target(debugserver
       COMMAND ${CMAKE_COMMAND} -E copy_if_different
       ${system_debugserver_path} ${LLVM_RUNTIME_OUTPUT_INTDIR}
       COMMENT "Copying the system debugserver to LLDB's binaries directory for testing.")
     # The custom target for the system debugserver has no install target, so we
     # need to remove it from the LLVM_DISTRIBUTION_COMPONENTS list.
     list(REMOVE_ITEM LLVM_DISTRIBUTION_COMPONENTS debugserver)
     set(LLVM_DISTRIBUTION_COMPONENTS ${LLVM_DISTRIBUTION_COMPONENTS} PARENT_SCOPE)
     message(STATUS "LLDB tests use out-of-tree debugserver: ${system_debugserver_path}")
-    list(APPEND LLDB_TEST_COMMON_ARGS --out-of-tree-debugserver)
+    list(APPEND LLDB_TEST_COMMON_ARGS --server ${system_debugserver_path})
   elseif(TARGET debugserver)
     set(debugserver_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/debugserver)
     message(STATUS "LLDB Tests use just-built debugserver: ${debugserver_path}")
     list(APPEND LLDB_TEST_COMMON_ARGS --server ${debugserver_path})
     add_dependencies(lldb-test-deps debugserver)
   elseif(TARGET lldb-server)
     set(lldb_server_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb-server)
     message(STATUS "LLDB Tests use just-built lldb-server: ${lldb_server_path}")