Index: packages/Python/lldbsuite/support/sockutil.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/support/sockutil.py @@ -0,0 +1,23 @@ +""" + The LLVM Compiler Infrastructure + +This file is distributed under the University of Illinois Open Source +License. See LICENSE.TXT for details. + +Helper functions for working with sockets. +""" + +# Python modules: +import io +import socket + +# LLDB modules +import use_lldb_suite + +def recvall(sock, size): + bytes = io.BytesIO() + while size > 0: + this_result = sock.recv(size) + bytes.write(this_result) + size -= len(this_result) + return bytes.getvalue() Index: scripts/swig_bot.py =================================================================== --- scripts/swig_bot.py +++ scripts/swig_bot.py @@ -1,11 +1,63 @@ #!/usr/bin/env python +""" +SWIG generation top-level script. Supports both local and remote generation +of SWIG bindings for multiple languages. +""" + # Python modules +import argparse +import logging import sys +import traceback # LLDB modules import use_lldb_suite +def process_args(args): + parser = argparse.ArgumentParser( + description='Run swig-bot client or server.') + + # Arguments to control whether swig-bot runs as a client or server. + parser.add_argument( + "--mode", + required=True, + choices=["client", "server"], + help="Run swig_bot in either client or server mode.") + + # Arguments to control logging verbosity. + parser.add_argument( + "--verbose", "-v", + action="store_true", + default=False, + help="Increase logging verbosity level.") + + (options, remaining) = parser.parse_known_args(args) + # Set logging level. + if options.verbose: + log_level = logging.DEBUG + else: + log_level = logging.NOTSET + logging.basicConfig(level=log_level) + logging.info("logging is using level: %d", log_level) + + return (options, remaining) + if __name__ == "__main__": - from swig_bot_lib import client - client.run(sys.argv[1:]) + (options, remaining) = process_args(sys.argv[1:]) + try: + if options.mode == "client": + logging.info("Running swig_bot in client mode") + from swig_bot_lib import client + client.run(remaining) + elif options.mode == "server": + logging.info("Running swig_bot in server mode") + from swig_bot_lib import server + server.run(remaining) + else: + logging.error("Unknown mode specified. Expected client or server.") + sys.exit(-1) + except Exception as e: + error = traceback.format_exc() + logging.error("An error occurred running swig-bot.") + logging.error(error) Index: scripts/swig_bot_lib/client.py =================================================================== --- scripts/swig_bot_lib/client.py +++ scripts/swig_bot_lib/client.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +""" +SWIG generation client. Supports both local and remote generation of SWIG +bindings for multiple languages. +""" + # Future imports from __future__ import absolute_import from __future__ import print_function @@ -8,11 +13,20 @@ import argparse import logging import os +import socket +import struct import sys # LLDB modules import use_lldb_suite from lldbsuite.support import fs +from lldbsuite.support import sockutil + +# package imports +from . import local + +default_ip = "127.0.0.1" +default_port = 8537 def process_args(args): """Returns options processed from the provided command line. @@ -20,9 +34,14 @@ @param args the command line to process. """ + # A custom action used by the --local command line option. It can be + # used with either 0 or 1 argument. If used with 0 arguments, it + # searches for a copy of swig located on the physical machine. If + # used with 1 argument, the argument is the path to a swig executable. class FindLocalSwigAction(argparse.Action): def __init__(self, option_strings, dest, **kwargs): - super(FindLocalSwigAction, self).__init__(option_strings, dest, nargs='?', **kwargs) + super(FindLocalSwigAction, self).__init__( + option_strings, dest, nargs='?', **kwargs) def __call__(self, parser, namespace, values, option_string=None): swig_exe = None if values is None: @@ -31,17 +50,33 @@ swig_exe = values setattr(namespace, self.dest, os.path.normpath(swig_exe)) + # A custom action used by the --remote command line option. It can be + # used with either 0 or 1 arguments. If used with 0 arguments it chooses + # a default connection string. If used with one argument it is a string + # of the form `ip_address[:port]`. If the port is unspecified, the + # default port is used. + class RemoteIpAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + super(RemoteIpAction, self).__init__( + option_strings, dest, nargs='?', **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + ip_port = None + if values is None: + ip_port = (default_ip, default_port) + else: + result = values.split(':') + if len(result)==1: + ip_port = (result[0], default_port) + elif len(result)==2: + ip_port = (result[0], int(result[1])) + else: + raise ValueError("Invalid connection string") + setattr(namespace, self.dest, ip_port) + # Setup the parser arguments that are accepted. parser = argparse.ArgumentParser( description='Generate SWIG bindings.') - # Arguments to control logging verbosity. - parser.add_argument( - "--verbose", "-v", - action="store_true", - default=False, - help="Increase logging verbosity level.") - parser.add_argument( "--local", action=FindLocalSwigAction, @@ -52,7 +87,7 @@ parser.add_argument( "--remote", - action="store", + action=RemoteIpAction, help=( "Use the given connection string to connect to a remote " "generation service")) @@ -85,24 +120,42 @@ logging.error("Must specify either --local or --remote") sys.exit(-3) - # Set logging level based on verbosity count. - if options.verbose: - log_level = logging.DEBUG - else: - log_level = logging.NOTSET - logging.basicConfig(level=log_level) - logging.info("logging is using level: %d", log_level) - return options +def establish_remote_connection(ip_port): + logging.debug("Creating socket...") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + logging.info("Connecting to server {} on port {}" + .format(ip_port[0], ip_port[1])) + s.connect(ip_port) + logging.info("Connection established...") + return s + +def transmit_data(connection, packed_input): + logging.info("Sending {} bytes of compressed data." + .format(len(packed_input))) + connection.sendall(struct.pack("!I", len(packed_input))) + connection.sendall(packed_input) + logging.info("Awaiting response.") + response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0] + logging.debug("Expecting {} byte response".format(response_len)) + response = sockutil.recvall(connection, response_len) + return response def run(args): options = process_args(args) if options.remote is None: + logging.info("swig bot client using local swig installation at '{}'" + .format(options.swig_executable)) if not os.path.isfile(options.swig_executable): - logging.error("Swig executable '%s' does not exist." % options.swig_executable) - from . import local + logging.error("Swig executable '{}' does not exist." + .format(options.swig_executable)) local.generate(options) else: - logging.error("Remote path is not yet implemented!") + logging.info("swig bot client using remote generation with server '{}'" + .format(options.remote)) + packed_input = local.pack_input(options) + connection = establish_remote_connection(options.remote) + response = transmit_data(connection, packed_input) + logging.debug("Received {} byte response.".format(len(response))) Index: scripts/swig_bot_lib/local.py =================================================================== --- scripts/swig_bot_lib/local.py +++ scripts/swig_bot_lib/local.py @@ -1,5 +1,10 @@ #!/usr/bin/env python +""" +Shared functionality used by `client` and `server` when generating or preparing +to generate SWIG on the local machine. +""" + # Future imports from __future__ import absolute_import from __future__ import print_function @@ -7,14 +12,57 @@ # Python modules import argparse import imp +import io import logging import os import subprocess import sys +import zipfile # LLDB modules import use_lldb_suite +def pack_input(options): + logging.info("Creating input file package...") + zip_data = io.BytesIO() + zip_file = None + try: + # It's possible that a custom-built interpreter will not have the + # standard zlib module. If so, we can only store, not compress. By + # try to compress since we usually have a standard Python distribution. + zip_file = zipfile.ZipFile(zip_data, mode='w', + compression=zipfile.ZIP_DEFLATED) + except RuntimeError: + zip_file = zipfile.ZipFile(zip_data, mode='w', + compression=zipfile.ZIP_STORED) + + filters = [("include/lldb", ".h"), + ("scripts", ".swig"), + ("scripts/Python", ".swig"), + ("scripts/interface", ".i")] + def filter_func(t): + subfolder = t[0] + ext = t[1] + full_path = os.path.normpath(os.path.join(options.src_root, subfolder)) + candidates = [os.path.normpath(os.path.join(full_path, f)) + for f in os.listdir(full_path)] + actual = filter( + lambda f : os.path.isfile(f) and os.path.splitext(f)[1] == ext, + candidates) + return (subfolder, map(lambda f : os.path.basename(f), actual)) + archive_entries = map(filter_func, filters) + + for entry in archive_entries: + subfolder = entry[0] + files = entry[1] + for file in files: + relative_path = os.path.normpath(os.path.join(subfolder, file)) + full_path = os.path.normpath( + os.path.join(options.src_root, relative_path)) + logging.info("{} -> {}".format(full_path, relative_path)) + zip_file.write(full_path, relative_path) + return zip_data.getvalue() + def generate(options): include_folder = os.path.join(options.src_root, "include") in_file = os.path.join(options.src_root, "scripts", "lldb.swig") @@ -43,7 +91,8 @@ in_file ]) - logging.info("generating swig {} bindings into {}".format(lang, out_dir)) + logging.info("generating swig {} bindings into {}" + .format(lang, out_dir)) logging.debug("swig command line: {}".format(swig_command)) try: # Execute swig @@ -54,5 +103,6 @@ if swig_output is not None and len(swig_output) > 0: logging.info("swig output: %s", swig_output) except subprocess.CalledProcessError as e: - logging.error("An error occurred executing swig. returncode={}".format(e.returncode)) + logging.error("An error occurred executing swig. returncode={}" + .format(e.returncode)) logging.error(e.output) Index: scripts/swig_bot_lib/server.py =================================================================== --- scripts/swig_bot_lib/server.py +++ scripts/swig_bot_lib/server.py @@ -1,6 +1,80 @@ #!/usr/bin/env python +""" +SWIG generation server. Listens for connections from swig generation clients +and runs swig in the requested fashion, sending back the results. +""" + # Future imports from __future__ import absolute_import from __future__ import print_function +# Python modules +import argparse +import logging +import os +import socket +import struct +import sys +import traceback + +# LLDB modules +import use_lldb_suite +from lldbsuite.support import sockutil + +# package imports +from . import local + +default_port = 8537 + +def process_args(args): + # Setup the parser arguments that are accepted. + parser = argparse.ArgumentParser(description='SWIG generation server.') + + parser.add_argument( + "--port", + action="store", + default=default_port, + help=("The local port to bind to")) + + # Process args. + return parser.parse_args(args) + +def initialize_listening_socket(options): + logging.debug("Creating socket...") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + logging.info("Binding to ip address '', port {}".format(options.port)) + s.bind(('', options.port)) + + logging.debug("Putting socket in listen mode...") + s.listen() + return s + +def accept_once(sock, options): + logging.debug("Waiting for connection...") + client, addr = sock.accept() + logging.info("Received connection from {}".format(addr)) + data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0] + logging.debug("Expecting {} bytes of data from client".format(data_size)) + data = sockutil.recvall(client, data_size) + logging.info("Received {} bytes of data from client".format(len(data))) + + logging.info("Sending {} byte response".format(len(data))) + client.sendall(struct.pack("!I", len(data))) + client.sendall(data) + +def accept_loop(sock, options): + while True: + try: + accept_once(sock, options) + except Exception as e: + error = traceback.format_exc() + logging.error("An error occurred while processing the connection.") + logging.error(error) + +def run(args): + options = process_args(args) + sock = initialize_listening_socket(options) + accept_loop(sock, options) + return options