swig_bot remote path preliminary implementation

[zturner]

Summary

With this patch, the client will package up all the required
inputs into a compressed zip file, establish a connection to the
server, send the input to the server, and wait for the server to
send a response (in this case the response is just echoed back to
the client).

This gets the network communication in place, and in a subsequent
patch I will follow up with the code that actually runs swig on
the server and sends back the output instead of echoing back the
input.

[zturner]

socket.recv doesn't guarantee a full read. The data can be truncated if it could not be read all at once. This patch fixes this by introducing a helper function in lldbsuite.support to read an exact number of bytes, retrying as necessary until it's complete. Then both the client and server are updated to use this function instead of socket.recv

[tfiala]

LGTM.

[tfiala]

Ready to close out?

packages/Python/lldbsuite/support/sockutil.py

+"""
+                     The LLVM Compiler Infrastructure
+
+This file is distributed under the University of Illinois Open Source
+License. See LICENSE.TXT for details.
+
+Helper functions for working with sockets.
+"""
+
+# Python modules:
+import io
+import socket
+
+# LLDB modules
+import use_lldb_suite
+
+def recvall(sock, size):
+    bytes = io.BytesIO()
+    while size > 0:
+        this_result = sock.recv(size)
+        bytes.write(this_result)
+        size -= len(this_result)
+    return bytes.getvalue()

scripts/swig_bot.py

 #!/usr/bin/env python
 
+"""
+SWIG generation top-level script.  Supports both local and remote generation
+of SWIG bindings for multiple languages.
+"""
+
 # Python modules
+import argparse
+import logging
 import sys
+import traceback
 
 # LLDB modules
 import use_lldb_suite
 
+def process_args(args):
+    parser = argparse.ArgumentParser(
+        description='Run swig-bot client or server.')
+
+    # Arguments to control whether swig-bot runs as a client or server.
+    parser.add_argument(
+        "--mode",
+        required=True,
+        choices=["client", "server"],
+        help="Run swig_bot in either client or server mode.")
+
+    # Arguments to control logging verbosity.
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        default=False,
+        help="Increase logging verbosity level.")
+
+    (options, remaining) = parser.parse_known_args(args)
+    # Set logging level.
+    if options.verbose:
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.NOTSET
+    logging.basicConfig(level=log_level)
+    logging.info("logging is using level: %d", log_level)
+
+    return (options, remaining)
+
 if __name__ == "__main__":
+    (options, remaining) = process_args(sys.argv[1:])
+    try:
+        if options.mode == "client":
+            logging.info("Running swig_bot in client mode")
             from swig_bot_lib import client
-    client.run(sys.argv[1:])
+            client.run(remaining)
+        elif options.mode == "server":
+            logging.info("Running swig_bot in server mode")
+            from swig_bot_lib import server
+            server.run(remaining)
+        else:
+            logging.error("Unknown mode specified.  Expected client or server.")
+            sys.exit(-1)
+    except Exception as e:
+        error = traceback.format_exc()
+        logging.error("An error occurred running swig-bot.")
+        logging.error(error)

scripts/swig_bot_lib/client.py

 #!/usr/bin/env python
 
+"""
+SWIG generation client.  Supports both local and remote generation of SWIG
+bindings for multiple languages.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function

 import argparse
 import logging
 import os
+import socket
+import struct
 import sys
 
 # LLDB modules
 import use_lldb_suite
 from lldbsuite.support import fs
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_ip = "127.0.0.1"
+default_port = 8537
 
 def process_args(args):
     """Returns options processed from the provided command line.

     @param args the command line to process.
     """
 
+    # A custom action used by the --local command line option.  It can be
+    # used with either 0 or 1 argument.  If used with 0 arguments, it
+    # searches for a copy of swig located on the physical machine.  If
+    # used with 1 argument, the argument is the path to a swig executable.
     class FindLocalSwigAction(argparse.Action):
         def __init__(self, option_strings, dest, **kwargs):
-            super(FindLocalSwigAction, self).__init__(option_strings, dest, nargs='?', **kwargs)
+            super(FindLocalSwigAction, self).__init__(
+                option_strings, dest, nargs='?', **kwargs)
         def __call__(self, parser, namespace, values, option_string=None):
             swig_exe = None
             if values is None:

                 swig_exe = values
             setattr(namespace, self.dest, os.path.normpath(swig_exe))
 
+    # A custom action used by the --remote command line option.  It can be
+    # used with either 0 or 1 arguments.  If used with 0 arguments it chooses
+    # a default connection string.  If used with one argument it is a string
+    # of the form `ip_address[:port]`.  If the port is unspecified, the
+    # default port is used.
+    class RemoteIpAction(argparse.Action):
+        def __init__(self, option_strings, dest, **kwargs):
+            super(RemoteIpAction, self).__init__(
+                option_strings, dest, nargs='?', **kwargs)
+        def __call__(self, parser, namespace, values, option_string=None):
+            ip_port = None
+            if values is None:
+                ip_port = (default_ip, default_port)
+            else:
+                result = values.split(':')
+                if len(result)==1:
+                    ip_port = (result[0], default_port)
+                elif len(result)==2:
+                    ip_port = (result[0], int(result[1]))
+                else:
+                    raise ValueError("Invalid connection string")
+            setattr(namespace, self.dest, ip_port)
+
     # Setup the parser arguments that are accepted.
     parser = argparse.ArgumentParser(
         description='Generate SWIG bindings.')
 
-    # Arguments to control logging verbosity.
-    parser.add_argument(
-        "--verbose", "-v",
-        action="store_true",
-        default=False,
-        help="Increase logging verbosity level.")
-
     parser.add_argument(
         "--local",
         action=FindLocalSwigAction,

 
     parser.add_argument(
         "--remote",
-        action="store",
+        action=RemoteIpAction,
         help=(
             "Use the given connection string to connect to a remote "
             "generation service"))

         logging.error("Must specify either --local or --remote")
         sys.exit(-3)
 
-    # Set logging level based on verbosity count.
-    if options.verbose:
-        log_level = logging.DEBUG
-    else:
-        log_level = logging.NOTSET
-    logging.basicConfig(level=log_level)
-    logging.info("logging is using level: %d", log_level)
-
     return options
 
+def establish_remote_connection(ip_port):
+    logging.debug("Creating socket...")
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    logging.info("Connecting to server {} on port {}"
+                 .format(ip_port[0], ip_port[1]))
+    s.connect(ip_port)
+    logging.info("Connection established...")
+    return s
+
+def transmit_data(connection, packed_input):
+    logging.info("Sending {} bytes of compressed data."
+                 .format(len(packed_input)))
+    connection.sendall(struct.pack("!I", len(packed_input)))
+    connection.sendall(packed_input)
+    logging.info("Awaiting response.")
+    response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0]
+    logging.debug("Expecting {} byte response".format(response_len))
+    response = sockutil.recvall(connection, response_len)
+    return response
 
 def run(args):
     options = process_args(args)
 
     if options.remote is None:
+        logging.info("swig bot client using local swig installation at '{}'"
+                     .format(options.swig_executable))
         if not os.path.isfile(options.swig_executable):
-            logging.error("Swig executable '%s' does not exist." % options.swig_executable)
-        from . import local
+            logging.error("Swig executable '{}' does not exist."
+                          .format(options.swig_executable))
         local.generate(options)
     else:
-        logging.error("Remote path is not yet implemented!")
+        logging.info("swig bot client using remote generation with server '{}'"
+                     .format(options.remote))
+        packed_input = local.pack_input(options)
+        connection = establish_remote_connection(options.remote)
+        response = transmit_data(connection, packed_input)
+        logging.debug("Received {} byte response.".format(len(response)))

scripts/swig_bot_lib/local.py

 #!/usr/bin/env python
 
+"""
+Shared functionality used by `client` and `server` when generating or preparing
+to generate SWIG on the local machine.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function

 # Python modules
 import argparse
 import imp
+import io
 import logging
 import os
 import subprocess
 import sys
+import zipfile
 
 # LLDB modules
 import use_lldb_suite
 
+def pack_input(options):
+    logging.info("Creating input file package...")
+    zip_data = io.BytesIO()
+    zip_file = None
+    try:
+        # It's possible that a custom-built interpreter will not have the
+        # standard zlib module.  If so, we can only store, not compress.  By
+        # try to compress since we usually have a standard Python distribution.
+        zip_file = zipfile.ZipFile(zip_data, mode='w',
+                                   compression=zipfile.ZIP_DEFLATED)
+    except RuntimeError:
+        zip_file = zipfile.ZipFile(zip_data, mode='w',
+                                   compression=zipfile.ZIP_STORED)
+
+    filters = [("include/lldb", ".h"),
+               ("scripts", ".swig"),
+               ("scripts/Python", ".swig"),
+               ("scripts/interface", ".i")]
+    def filter_func(t):
+        subfolder = t[0]
+        ext = t[1]
+        full_path = os.path.normpath(os.path.join(options.src_root, subfolder))
+        candidates = [os.path.normpath(os.path.join(full_path, f))
+                      for f in os.listdir(full_path)]
+        actual = filter(
+            lambda f : os.path.isfile(f) and os.path.splitext(f)[1] == ext,
+            candidates)
+        return (subfolder, map(lambda f : os.path.basename(f), actual))
+    archive_entries = map(filter_func, filters)
+
+    for entry in archive_entries:
+        subfolder = entry[0]
+        files = entry[1]
+        for file in files:
+            relative_path = os.path.normpath(os.path.join(subfolder, file))
+            full_path = os.path.normpath(
+                os.path.join(options.src_root, relative_path))
+            logging.info("{} -> {}".format(full_path, relative_path))
+            zip_file.write(full_path, relative_path)
+    return zip_data.getvalue()
+
 def generate(options):
     include_folder = os.path.join(options.src_root, "include")
     in_file = os.path.join(options.src_root, "scripts", "lldb.swig")

             in_file
         ])
 
-        logging.info("generating swig {} bindings into {}".format(lang, out_dir))
+        logging.info("generating swig {} bindings into {}"
+                     .format(lang, out_dir))
         logging.debug("swig command line: {}".format(swig_command))
         try:
             # Execute swig

             if swig_output is not None and len(swig_output) > 0:
                 logging.info("swig output: %s", swig_output)
         except subprocess.CalledProcessError as e:
-            logging.error("An error occurred executing swig.  returncode={}".format(e.returncode))
+            logging.error("An error occurred executing swig.  returncode={}"
+                          .format(e.returncode))
             logging.error(e.output)

scripts/swig_bot_lib/server.py

 #!/usr/bin/env python
 
+"""
+SWIG generation server.  Listens for connections from swig generation clients
+and runs swig in the requested fashion, sending back the results.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function
 
+# Python modules
+import argparse
+import logging
+import os
+import socket
+import struct
+import sys
+import traceback
+
+# LLDB modules
+import use_lldb_suite
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_port = 8537
+
+def process_args(args):
+    # Setup the parser arguments that are accepted.
+    parser = argparse.ArgumentParser(description='SWIG generation server.')
+
+    parser.add_argument(
+        "--port",
+        action="store",
+        default=default_port,
+        help=("The local port to bind to"))
+
+    # Process args.
+    return parser.parse_args(args)
+
+def initialize_listening_socket(options):
+    logging.debug("Creating socket...")
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+    logging.info("Binding to ip address '', port {}".format(options.port))
+    s.bind(('', options.port))
+
+    logging.debug("Putting socket in listen mode...")
+    s.listen()
+    return s
+
+def accept_once(sock, options):
+    logging.debug("Waiting for connection...")
+    client, addr = sock.accept()
+    logging.info("Received connection from {}".format(addr))
+    data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0]
+    logging.debug("Expecting {} bytes of data from client".format(data_size))
+    data = sockutil.recvall(client, data_size)
+    logging.info("Received {} bytes of data from client".format(len(data)))
+
+    logging.info("Sending {} byte response".format(len(data)))
+    client.sendall(struct.pack("!I", len(data)))
+    client.sendall(data)
+
+def accept_loop(sock, options):
+    while True:
+        try:
+            accept_once(sock, options)
+        except Exception as e:
+            error = traceback.format_exc()
+            logging.error("An error occurred while processing the connection.")
+            logging.error(error)
+
+def run(args):
+    options = process_args(args)
+    sock = initialize_listening_socket(options)
+    accept_loop(sock, options)
+    return options