Preparation for making a Python package out of lldb's Python code

[zturner]

Summary

See the thread in lldb-dev for context.

After this patch, the remaining piece of work is to literally move lldb/test to lldb/packages/Python/lldbsuite/test and apply 1 or 2 fixups at places marked in this patch.

[zturner]

Moved progress.py from lldb/test to lldb/third_party. This seems to be a third party module that had also snuck into the test folder. It's used by both unittest2 and dosep.py so it has to be in third_party to be usable from both.

[zturner]

Fixed lldbsuite module initialization. I'm having to do this all by eye / memory since this code isn't actually exercised yet until the move actually happens.

[tfiala]

LGTM.

lldb/trunk/packages/Python/lldbsuite/__init__.py

+# Module level initialization for the `lldbsuite` module.
+
+import inspect
+import os
+import sys
+
+def find_lldb_root():
+    lldb_root = os.path.dirname(inspect.getfile(inspect.currentframe()))
+    while True:
+        lldb_root = os.path.dirname(lldb_root)
+        if lldb_root is None:
+            return None
+
+        test_path = os.path.join(lldb_root, "lldb.root")
+        if os.path.isfile(test_path):
+            return lldb_root
+    return None
+
+# lldbsuite.lldb_root refers to the root of the git/svn source checkout
+lldb_root = find_lldb_root()

lldb/trunk/packages/Python/lldbsuite/test/__init__.py

+# Module level initialization for the `lldbsuite.test` module.

lldb/trunk/test/dosep.py

def adjust_inferior_options(dotest_argv):
     global dotest_options
 
     # If we don't have a session directory, create one.
     if not dotest_options.s:
         # no session log directory, we need to add this to prevent
         # every dotest invocation from creating its own directory
         import datetime
         # The windows platforms don't like ':' in the pathname.
-        timestamp_started = datetime.datetime.now().strftime("%F-%H_%M_%S")
+        timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
         dotest_argv.append('-s')
         dotest_argv.append(timestamp_started)
         dotest_options.s = timestamp_started
 
     # Adjust inferior results formatter options - if the parallel
     # test runner is collecting into the user-specified test results,
     # we'll have inferiors spawn with the --results-port option and
     # strip the original test runner options.

lldb/trunk/test/dotest.py

if rdir:
             os.environ["LLDB_TEST"] = os.path.join(rdir, 'test')
         else:
             os.environ["LLDB_TEST"] = rdir
     else:
         os.environ["LLDB_TEST"] = scriptPath
 
     # Set up the LLDB_SRC environment variable, so that the tests can locate
     # the LLDB source code.
+    # When this changes over to a package instead of a standalone script, this
+    # will be `lldbsuite.lldb_root`
     os.environ["LLDB_SRC"] = os.path.join(scriptPath, os.pardir)
 
     pluginPath = os.path.join(scriptPath, 'plugins')
     toolsLLDBMIPath = os.path.join(scriptPath, 'tools', 'lldb-mi')
     toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server')
 
     # Insert script dir, plugin dir, lldb-mi dir and lldb-server dir to the sys.path.
     sys.path.insert(0, scriptPath)
     sys.path.insert(0, pluginPath)
     sys.path.insert(0, toolsLLDBMIPath)      # Adding test/tools/lldb-mi to the path makes it easy
                                              # to "import lldbmi_testcase" from the MI tests
     sys.path.insert(0, toolsLLDBServerPath)  # Adding test/tools/lldb-server to the path makes it easy
                                              # to "import lldbgdbserverutils" from the lldb-server tests
 
     # This is the root of the lldb git/svn checkout
+    # When this changes over to a package instead of a standalone script, this
+    # will be `lldbsuite.lldb_root`
     lldbRootDirectory = os.path.abspath(os.path.join(scriptPath, os.pardir))
 
     # Some of the tests can invoke the 'lldb' command directly.
     # We'll try to locate the appropriate executable right here.
 
     # The lldb executable can be set from the command line
     # if it's not set, we try to find it now
     # first, we try the environment
for name in names:
             if not filterspec or not filtered:
                 # A simple case of just the module name.  Also the failover case
                 # from the filterspec branch when the (base, filterspec) combo
                 # doesn't make sense.
                 suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
 
 
 def disabledynamics():
+    import lldb
     ci = lldb.DBG.GetCommandInterpreter()
     res = lldb.SBCommandReturnObject()
     ci.HandleCommand("setting set target.prefer-dynamic-value no-dynamic-values", res, False)    
     if not res.Succeeded():
         raise Exception('disabling dynamic type support failed')
 
 def lldbLoggings():
+    import lldb
     """Check and do lldb loggings if necessary."""
 
     # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is
     # defined.  Use ${LLDB_LOG} to specify the log file.
     ci = lldb.DBG.GetCommandInterpreter()
     res = lldb.SBCommandReturnObject()
     if ("LLDB_LOG" in os.environ):
         open(os.environ["LLDB_LOG"], 'w').close()
def checkDsymForUUIDIsNotOn():
     if cmd_output and "DBGFileMappedPaths = " in cmd_output:
         print("%s =>" % ' '.join(cmd))
         print(cmd_output)
         print("Disable automatic lookup and caching of dSYMs before running the test suite!")
         print("Exiting...")
         sys.exit(0)
 
 def exitTestSuite(exitCode = None):
+    import lldb
     lldb.SBDebugger.Terminate()
     if exitCode:
         sys.exit(exitCode)
 
 
 def isMultiprocessTestRunner():
     # We're not multiprocess when we're either explicitly
     # the inferior (as specified by the multiprocess test
     # runner) OR we've been told to skip using the multiprocess
     # test runner
     return not (is_inferior_test_runner or no_multiprocess_test_runner)
 
-if __name__ == "__main__":
+def run_suite():
+    global just_do_benchmarks_test
+    global dont_do_dsym_test
+    global dont_do_dwarf_test
+    global dont_do_dwo_test
+    global blacklist
+    global blacklistConfig
+    global categoriesList
+    global validCategories
+    global useCategories
+    global skipCategories
+    global lldbFrameworkPath
+    global configFile
+    global archs
+    global compilers
+    global count
+    global dumpSysPath
+    global bmExecutable
+    global bmBreakpointSpec
+    global bmIterationCount
+    global failed
+    global failfast
+    global filters
+    global fs4all
+    global ignore
+    global progress_bar
+    global runHooks
+    global skip_build_and_cleanup
+    global skip_long_running_test
+    global noHeaders
+    global parsable
+    global regexp
+    global rdir
+    global sdir_name
+    global svn_silent
+    global verbose
+    global testdirs
+    global lldb_platform_name
+    global lldb_platform_url
+    global lldb_platform_working_dir
+    global setCrashInfoHook
+    global is_inferior_test_runner
+    global multiprocess_test_subdir
+    global num_threads
+    global output_on_success
+    global no_multiprocess_test_runner
+    global test_runner_name
+    global results_filename
+    global results_formatter_name
+    global results_formatter_options
+    global results_port
+
     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
     # does not exist before proceeding to running the test suite.
     if sys.platform.startswith("darwin"):
         checkDsymForUUIDIsNotOn()
 
     #
     # Start the actions by first parsing the options while setting up the test
     # directories, followed by setting up the search paths for lldb utilities;
def run_suite():
     # Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined.
     # This should not be necessary now.
     if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ):
         print("Terminating Test suite...")
         subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())])
 
     # Exiting.
     exitTestSuite(failed)
+
+if __name__ == "__main__":
+    run_suite()
 No newline at end of file

lldb/trunk/test/progress.py

-#!/usr/bin/python
-
-from __future__ import print_function
-
-import use_lldb_suite
-import six
-
-import sys
-import time
-
-class ProgressBar(object):
-    """ProgressBar class holds the options of the progress bar.
-    The options are:
-        start   State from which start the progress. For example, if start is 
-                5 and the end is 10, the progress of this state is 50%
-        end     State in which the progress has terminated.
-        width   --
-        fill    String to use for "filled" used to represent the progress
-        blank   String to use for "filled" used to represent remaining space.
-        format  Format
-        incremental
-    """
-    light_block = six.unichr(0x2591).encode("utf-8")
-    solid_block = six.unichr(0x2588).encode("utf-8")
-    solid_right_arrow = six.unichr(0x25BA).encode("utf-8")
-    
-    def __init__(self, 
-                 start=0, 
-                 end=10, 
-                 width=12, 
-                 fill=six.unichr(0x25C9).encode("utf-8"), 
-                 blank=six.unichr(0x25CC).encode("utf-8"), 
-                 marker=six.unichr(0x25CE).encode("utf-8"), 
-                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
-                 incremental=True):
-        super(ProgressBar, self).__init__()
-
-        self.start = start
-        self.end = end
-        self.width = width
-        self.fill = fill
-        self.blank = blank
-        self.marker = marker
-        self.format = format
-        self.incremental = incremental
-        self.step = 100 / float(width) #fix
-        self.reset()
-
-    def __add__(self, increment):
-        increment = self._get_progress(increment)
-        if 100 > self.progress + increment:
-            self.progress += increment
-        else:
-            self.progress = 100
-        return self
-
-    def complete(self):
-        self.progress = 100
-        return self
-
-    def __str__(self):
-        progressed = int(self.progress / self.step) #fix
-        fill = progressed * self.fill
-        blank = (self.width - progressed) * self.blank
-        return self.format % {'fill': fill, 'blank': blank, 'marker': self.marker, 'progress': int(self.progress)}
-
-    __repr__ = __str__
-
-    def _get_progress(self, increment):
-        return float(increment * 100) / self.end
-
-    def reset(self):
-        """Resets the current progress to the start point"""
-        self.progress = self._get_progress(self.start)
-        return self
-
-
-class AnimatedProgressBar(ProgressBar):
-    """Extends ProgressBar to allow you to use it straighforward on a script.
-    Accepts an extra keyword argument named `stdout` (by default use sys.stdout)
-    and may be any file-object to which send the progress status.
-    """
-    def __init__(self, 
-                 start=0, 
-                 end=10, 
-                 width=12, 
-                 fill=six.unichr(0x25C9).encode("utf-8"), 
-                 blank=six.unichr(0x25CC).encode("utf-8"), 
-                 marker=six.unichr(0x25CE).encode("utf-8"), 
-                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
-                 incremental=True,
-                 stdout=sys.stdout):
-        super(AnimatedProgressBar, self).__init__(start,end,width,fill,blank,marker,format,incremental)
-        self.stdout = stdout
-
-    def show_progress(self):
-        if hasattr(self.stdout, 'isatty') and self.stdout.isatty():
-            self.stdout.write('\r')
-        else:
-            self.stdout.write('\n')
-        self.stdout.write(str(self))
-        self.stdout.flush()
-
-class ProgressWithEvents(AnimatedProgressBar):
-    """Extends AnimatedProgressBar to allow you to track a set of events that
-       cause the progress to move. For instance, in a deletion progress bar, you
-       can track files that were nuked and files that the user doesn't have access to
-    """
-    def __init__(self, 
-                 start=0, 
-                 end=10, 
-                 width=12, 
-                 fill=six.unichr(0x25C9).encode("utf-8"), 
-                 blank=six.unichr(0x25CC).encode("utf-8"), 
-                 marker=six.unichr(0x25CE).encode("utf-8"), 
-                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
-                 incremental=True,
-                 stdout=sys.stdout):
-        super(ProgressWithEvents, self).__init__(start,end,width,fill,blank,marker,format,incremental,stdout)
-        self.events = {}
-
-    def add_event(self,event):
-        if event in self.events:
-            self.events[event] += 1
-        else:
-            self.events[event] = 1
-
-    def show_progress(self):
-        isatty = hasattr(self.stdout, 'isatty') and self.stdout.isatty()
-        if isatty:
-            self.stdout.write('\r')
-        else:
-            self.stdout.write('\n')
-        self.stdout.write(str(self))
-        if len(self.events) == 0:
-            return
-        self.stdout.write('\n')
-        for key in list(self.events.keys()):
-            self.stdout.write(str(key) + ' = ' + str(self.events[key]) + ' ')
-        if isatty:
-            self.stdout.write('\033[1A')
-        self.stdout.flush()
-
-
-if __name__ == '__main__':
-    p = AnimatedProgressBar(end=200, width=200)
-
-    while True:
-        p + 5
-        p.show_progress()
-        time.sleep(0.3)
-        if p.progress == 100:
-            break
-    print() #new line
 

lldb/trunk/test/use_lldb_suite.py

while True:
             return lldb_root
     return None
 
 lldb_root = find_lldb_root()
 if lldb_root is not None:
     import imp
     module = imp.find_module("use_lldb_suite_root", [lldb_root])
     if module is not None:
-        imp.load_module("use_lldb_suite_root", *module)
+        imp.load_module("use_lldb_suite_root", *module)
 

lldb/trunk/third_party/Python/module/progress/progress.py

+#!/usr/bin/python
+
+from __future__ import print_function
+
+import use_lldb_suite
+import six
+
+import sys
+import time
+
+class ProgressBar(object):
+    """ProgressBar class holds the options of the progress bar.
+    The options are:
+        start   State from which start the progress. For example, if start is 
+                5 and the end is 10, the progress of this state is 50%
+        end     State in which the progress has terminated.
+        width   --
+        fill    String to use for "filled" used to represent the progress
+        blank   String to use for "filled" used to represent remaining space.
+        format  Format
+        incremental
+    """
+    light_block = six.unichr(0x2591).encode("utf-8")
+    solid_block = six.unichr(0x2588).encode("utf-8")
+    solid_right_arrow = six.unichr(0x25BA).encode("utf-8")
+    
+    def __init__(self, 
+                 start=0, 
+                 end=10, 
+                 width=12, 
+                 fill=six.unichr(0x25C9).encode("utf-8"), 
+                 blank=six.unichr(0x25CC).encode("utf-8"), 
+                 marker=six.unichr(0x25CE).encode("utf-8"), 
+                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
+                 incremental=True):
+        super(ProgressBar, self).__init__()
+
+        self.start = start
+        self.end = end
+        self.width = width
+        self.fill = fill
+        self.blank = blank
+        self.marker = marker
+        self.format = format
+        self.incremental = incremental
+        self.step = 100 / float(width) #fix
+        self.reset()
+
+    def __add__(self, increment):
+        increment = self._get_progress(increment)
+        if 100 > self.progress + increment:
+            self.progress += increment
+        else:
+            self.progress = 100
+        return self
+
+    def complete(self):
+        self.progress = 100
+        return self
+
+    def __str__(self):
+        progressed = int(self.progress / self.step) #fix
+        fill = progressed * self.fill
+        blank = (self.width - progressed) * self.blank
+        return self.format % {'fill': fill, 'blank': blank, 'marker': self.marker, 'progress': int(self.progress)}
+
+    __repr__ = __str__
+
+    def _get_progress(self, increment):
+        return float(increment * 100) / self.end
+
+    def reset(self):
+        """Resets the current progress to the start point"""
+        self.progress = self._get_progress(self.start)
+        return self
+
+
+class AnimatedProgressBar(ProgressBar):
+    """Extends ProgressBar to allow you to use it straighforward on a script.
+    Accepts an extra keyword argument named `stdout` (by default use sys.stdout)
+    and may be any file-object to which send the progress status.
+    """
+    def __init__(self, 
+                 start=0, 
+                 end=10, 
+                 width=12, 
+                 fill=six.unichr(0x25C9).encode("utf-8"), 
+                 blank=six.unichr(0x25CC).encode("utf-8"), 
+                 marker=six.unichr(0x25CE).encode("utf-8"), 
+                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
+                 incremental=True,
+                 stdout=sys.stdout):
+        super(AnimatedProgressBar, self).__init__(start,end,width,fill,blank,marker,format,incremental)
+        self.stdout = stdout
+
+    def show_progress(self):
+        if hasattr(self.stdout, 'isatty') and self.stdout.isatty():
+            self.stdout.write('\r')
+        else:
+            self.stdout.write('\n')
+        self.stdout.write(str(self))
+        self.stdout.flush()
+
+class ProgressWithEvents(AnimatedProgressBar):
+    """Extends AnimatedProgressBar to allow you to track a set of events that
+       cause the progress to move. For instance, in a deletion progress bar, you
+       can track files that were nuked and files that the user doesn't have access to
+    """
+    def __init__(self, 
+                 start=0, 
+                 end=10, 
+                 width=12, 
+                 fill=six.unichr(0x25C9).encode("utf-8"), 
+                 blank=six.unichr(0x25CC).encode("utf-8"), 
+                 marker=six.unichr(0x25CE).encode("utf-8"), 
+                 format='[%(fill)s%(marker)s%(blank)s] %(progress)s%%', 
+                 incremental=True,
+                 stdout=sys.stdout):
+        super(ProgressWithEvents, self).__init__(start,end,width,fill,blank,marker,format,incremental,stdout)
+        self.events = {}
+
+    def add_event(self,event):
+        if event in self.events:
+            self.events[event] += 1
+        else:
+            self.events[event] = 1
+
+    def show_progress(self):
+        isatty = hasattr(self.stdout, 'isatty') and self.stdout.isatty()
+        if isatty:
+            self.stdout.write('\r')
+        else:
+            self.stdout.write('\n')
+        self.stdout.write(str(self))
+        if len(self.events) == 0:
+            return
+        self.stdout.write('\n')
+        for key in list(self.events.keys()):
+            self.stdout.write(str(key) + ' = ' + str(self.events[key]) + ' ')
+        if isatty:
+            self.stdout.write('\033[1A')
+        self.stdout.flush()
+
+
+if __name__ == '__main__':
+    p = AnimatedProgressBar(end=200, width=200)
+
+    while True:
+        p + 5
+        p.show_progress()
+        time.sleep(0.3)
+        if p.progress == 100:
+            break
+    print() #new line
 No newline at end of file

lldb/trunk/use_lldb_suite_root.py

 import inspect
 import os
 import sys
 
 def add_third_party_module_dirs(lldb_root):
     third_party_modules_dir = os.path.join(lldb_root, "third_party", "Python", "module")
     if not os.path.isdir(third_party_modules_dir):
         return
 
     module_dirs = os.listdir(third_party_modules_dir)
     for module_dir in module_dirs:
         module_dir = os.path.join(third_party_modules_dir, module_dir)
         sys.path.insert(0, module_dir)
+
+def add_lldbsuite_packages_dir(lldb_root):
+    packages_dir = os.path.join(lldb_root, "packages", "Python")
+    sys.path.insert(0, packages_dir)
+
 lldb_root = os.path.dirname(inspect.getfile(inspect.currentframe()))
+
 add_third_party_module_dirs(lldb_root)
+add_lldbsuite_packages_dir(lldb_root)