[lldb] Parallelize fetching symbol files in crashlog.py

[JDevlieghere]

Summary

When using dsymForUUID, the majority of time symbolication a crashlog with crashlog.py is spent waiting for it to complete. Currently, we're calling dsymForUUID sequentially when iterating over the modules. We can drastically cut down this time by calling dsymForUUID in parallel. This patch uses Python's ThreadPoolExecutor (introduced in Python 3.2) to parallelize this IO-bound operation.

The performance improvement is hard to benchmark, because even with an empty local cache, consecutive calls to dsymForUUID for the same UUID complete faster. With warm caches, I'm seeing a ~30% performance improvement (~90s -> ~60s). I suspect the gains will be much bigger for a cold cache.

[JDevlieghere]

lldb/examples/python/crashlog.py

272

The changes to this line and the one below are the result of "Getting symbols [...]" and "Resolved symbols [...]" no longer appearing after each other. By parallelizing this operation you get a bunch of consecutive "Getting symbols [...]" followed by a bunch of "Resolved symbols [...]" lines.

[mib]

Very cool! LGTM!

[clayborg]

lgtm!

[JDevlieghere]

a8abb695859ad4e7fe695b9ee238a2b0cd00af7c

lldb/examples/python/crashlog.py

 #
 # On MacOSX csh, tcsh:
 #   ( setenv PYTHONPATH /path/to/LLDB.framework/Resources/Python ; ./crashlog.py ~/Library/Logs/DiagnosticReports/a.crash )
 #
 # On MacOSX sh, bash:
 #   PYTHONPATH=/path/to/LLDB.framework/Resources/Python ./crashlog.py ~/Library/Logs/DiagnosticReports/a.crash
 #----------------------------------------------------------------------
 
-from __future__ import print_function
 import cmd
 import contextlib
 import datetime
 import glob
 import json
 import optparse
 import os
 import platform
 import plistlib
 import re
 import shlex
 import string
 import subprocess
 import sys
 import time
 import uuid
+import concurrent.futures
 
 try:
     # First try for LLDB in case PYTHONPATH is already correctly setup.
     import lldb
 except ImportError:
     # Ask the command line driver for the path to the lldb module. Copy over
     # the environment so that SDKROOT is propagated to xcrun.
     command =  ['xcrun', 'lldb', '-P'] if platform.system() == 'Darwin' else ['lldb', '-P']
class DarwinImage(symbolication.Image):
         def locate_module_and_debug_symbols(self):
             # Don't load a module twice...
             if self.resolved:
                 return True
             # Mark this as resolved so we don't keep trying
             self.resolved = True
             uuid_str = self.get_normalized_uuid_string()
             if self.show_symbol_progress():
-                print('Getting symbols for %s %s...\n' % (uuid_str, self.path), end=' ')
+                print('Getting symbols for %s %s...' % (uuid_str, self.path))
             if os.path.exists(self.dsymForUUIDBinary):
                 dsym_for_uuid_command = '%s %s' % (
                     self.dsymForUUIDBinary, uuid_str)
                 s = subprocess.check_output(dsym_for_uuid_command, shell=True)
                 if s:
                     try:
                         plist_root = read_plist(s)
                     except:
class DarwinImage(symbolication.Image):
                               found_matching_slice = True
                               break
                         if found_matching_slice:
                            break
                 except:
                     pass
             if (self.resolved_path and os.path.exists(self.resolved_path)) or (
                     self.path and os.path.exists(self.path)):
-                print('Resolved symbols for %s %s...\n' % (uuid_str, self.path), end=' ')
+                print('Resolved symbols for %s %s...' % (uuid_str, self.path))
                 return True
             else:
                 self.unavailable = True
             return False
 
     def __init__(self, debugger, path, verbose):
         """CrashLog constructor that take a path to a darwin crash log file"""
         symbolication.Symbolicator.__init__(self, debugger)
class Symbolicate:
 
     def get_short_help(self):
         return "Symbolicate one or more darwin crash log files."
 
     def get_long_help(self):
         option_parser = CrashLogOptionParser()
         return option_parser.format_help()
 
-
 def SymbolicateCrashLog(crash_log, options):
     if options.debug:
         crash_log.dump()
     if not crash_log.images:
         print('error: no images in crash log')
         return
 
     if options.dump_image_list:
else:
             for ident in crash_log.idents:
                 images = crash_log.find_images_with_identifier(ident)
                 if images:
                     for image in images:
                         images_to_load.append(image)
                 else:
                     print('error: can\'t find image for identifier "%s"' % ident)
 
+    futures = []
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        def add_module(image, target):
+            return image, image.add_module(target)
+
         for image in images_to_load:
-        if image not in loaded_images:
-            err = image.add_module(target)
+            futures.append(executor.submit(add_module, image=image, target=target))
+
+        for future in concurrent.futures.as_completed(futures):
+            image, err = future.result()
             if err:
                 print(err)
             else:
                 loaded_images.append(image)
 
     if crash_log.backtraces:
         for thread in crash_log.backtraces:
             thread.dump_symbolicated(crash_log, options)