Index: lldb/scripts/swig_bot.py =================================================================== --- lldb/scripts/swig_bot.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python - -""" -SWIG generation top-level script. Supports both local and remote generation -of SWIG bindings for multiple languages. -""" - -# Python modules -import argparse -import logging -import sys -import traceback - -# LLDB modules -import use_lldb_suite - -# swig_bot modules -from swig_bot_lib import client -from swig_bot_lib import server - - -def process_args(args): - parser = argparse.ArgumentParser( - description='Run swig-bot client or server.') - - # Create and populate subparser arguments for when swig_bot is - # run in client or server mode - subparsers = parser.add_subparsers( - help="Pass --help to a sub-command to print detailed usage") - client_parser = subparsers.add_parser("client", - help="Run SWIG generation client") - client.add_subparser_args(client_parser) - client_parser.set_defaults(func=run_client) - - server_parser = subparsers.add_parser("server", - help="Run SWIG generation server") - server.add_subparser_args(server_parser) - server_parser.set_defaults(func=run_server) - - # Arguments to control logging verbosity. - parser.add_argument( - "--verbose", "-v", - action="store_true", - default=False, - help="Increase logging verbosity level.") - - options = parser.parse_args(args) - # Set logging level. - if options.verbose: - log_level = logging.DEBUG - else: - log_level = logging.NOTSET - logging.basicConfig(level=log_level) - logging.info("logging is using level: %d", log_level) - - return options - - -def run_client(options): - logging.info("Running swig_bot in client mode") - client.finalize_subparser_options(options) - client.run(options) - - -def run_server(options): - logging.info("Running swig_bot in server mode") - server.finalize_subparser_options(options) - server.run(options) - -if __name__ == "__main__": - options = process_args(sys.argv[1:]) - try: - if options.func is None: - logging.error( - "Unknown mode specified. Expected client or server.") - sys.exit(-1) - else: - options.func(options) - except KeyboardInterrupt as e: - logging.info("Ctrl+C received. Shutting down...") - sys.exit(-1) - except Exception as e: - error = traceback.format_exc() - logging.error("An error occurred running swig-bot.") - logging.error(error) Index: lldb/scripts/swig_bot_lib/client.py =================================================================== --- lldb/scripts/swig_bot_lib/client.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python - -""" -SWIG generation client. Supports both local and remote generation of SWIG -bindings for multiple languages. -""" - -# Future imports -from __future__ import absolute_import -from __future__ import print_function - -# Python modules -import argparse -import io -import logging -import os -import socket -import struct -import sys - -# LLDB modules -import use_lldb_suite -from lldbsuite.support import fs -from lldbsuite.support import sockutil - -# package imports -from . import local -from . import remote - -default_ip = "127.0.0.1" -default_port = 8537 - - -def add_subparser_args(parser): - """Returns options processed from the provided command line. - - @param args the command line to process. - """ - - # A custom action used by the --local command line option. It can be - # used with either 0 or 1 argument. If used with 0 arguments, it - # searches for a copy of swig located on the physical machine. If - # used with 1 argument, the argument is the path to a swig executable. - class FindLocalSwigAction(argparse.Action): - - def __init__(self, option_strings, dest, **kwargs): - super(FindLocalSwigAction, self).__init__( - option_strings, dest, nargs='?', **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - swig_exe = None - if values is None: - swig_exe = fs.find_executable('swig') - else: - swig_exe = values - setattr(namespace, self.dest, os.path.normpath(swig_exe)) - - # A custom action used by the --remote command line option. It can be - # used with either 0 or 1 arguments. If used with 0 arguments it chooses - # a default connection string. If used with one argument it is a string - # of the form `ip_address[:port]`. If the port is unspecified, the - # default port is used. - class RemoteIpAction(argparse.Action): - - def __init__(self, option_strings, dest, **kwargs): - super(RemoteIpAction, self).__init__( - option_strings, dest, nargs='?', **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - ip_port = None - if values is None: - ip_port = (default_ip, default_port) - else: - result = values.split(':') - if len(result) == 1: - ip_port = (result[0], default_port) - elif len(result) == 2: - ip_port = (result[0], int(result[1])) - else: - raise ValueError("Invalid connection string") - setattr(namespace, self.dest, ip_port) - - parser.add_argument( - "--local", - action=FindLocalSwigAction, - dest="swig_executable", - help=( - "Run the copy of swig at the specified location, or search PATH" - "if the location is omitted")) - - parser.add_argument( - "--remote", - action=RemoteIpAction, - help=( - "Use the given connection string to connect to a remote " - "generation service")) - - parser.add_argument( - "--src-root", - required=True, - help="The root folder of the LLDB source tree.") - - parser.add_argument( - "--target-dir", - default=os.getcwd(), - help=( - "Specifies the build dir where the language binding " - "should be placed")) - - parser.add_argument( - "--language", - dest="languages", - action="append", - help="Specifies the language to generate bindings for") - - -def finalize_subparser_options(options): - if options.languages is None: - options.languages = ['python'] - - if options.remote is None and options.swig_executable is None: - logging.error("Must specify either --local or --remote") - sys.exit(-3) - - return options - - -def establish_remote_connection(ip_port): - logging.debug("Creating socket...") - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - logging.info("Connecting to server {} on port {}" - .format(ip_port[0], ip_port[1])) - s.connect(ip_port) - logging.info("Connection established...") - return s - - -def transmit_request(connection, packed_input): - logging.info("Sending {} bytes of compressed data." - .format(len(packed_input))) - connection.sendall(struct.pack("!I", len(packed_input))) - connection.sendall(packed_input) - logging.info("Awaiting response.") - response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0] - logging.debug("Expecting {} byte response".format(response_len)) - response = sockutil.recvall(connection, response_len) - return response - - -def handle_response(options, connection, response): - logging.debug("Received {} byte response.".format(len(response))) - logging.debug("Creating output directory {}" - .format(options.target_dir)) - os.makedirs(options.target_dir, exist_ok=True) - - logging.info("Unpacking response archive into {}" - .format(options.target_dir)) - local.unpack_archive(options.target_dir, response) - response_file_path = os.path.normpath( - os.path.join(options.target_dir, "swig_output.json")) - if not os.path.isfile(response_file_path): - logging.error("Response file '{}' does not exist." - .format(response_file_path)) - return - try: - response = remote.deserialize_response_status( - io.open(response_file_path)) - if response[0] != 0: - logging.error("An error occurred during generation. Status={}" - .format(response[0])) - logging.error(response[1]) - else: - logging.info("SWIG generation successful.") - if len(response[1]) > 0: - logging.info(response[1]) - finally: - os.unlink(response_file_path) - - -def run(options): - if options.remote is None: - logging.info("swig bot client using local swig installation at '{}'" - .format(options.swig_executable)) - if not os.path.isfile(options.swig_executable): - logging.error("Swig executable '{}' does not exist." - .format(options.swig_executable)) - config = local.LocalConfig() - config.languages = options.languages - config.src_root = options.src_root - config.target_dir = options.target_dir - config.swig_executable = options.swig_executable - local.generate(config) - else: - logging.info("swig bot client using remote generation with server '{}'" - .format(options.remote)) - connection = None - try: - config = remote.generate_config(options.languages) - logging.debug("Generated config json {}".format(config)) - inputs = [("include/lldb", ".h"), - ("include/lldb/API", ".h"), - ("scripts", ".swig"), - ("scripts/Python", ".swig"), - ("scripts/interface", ".i")] - zip_data = io.BytesIO() - packed_input = local.pack_archive( - zip_data, options.src_root, inputs) - logging.info("(null) -> config.json") - packed_input.writestr("config.json", config) - packed_input.close() - connection = establish_remote_connection(options.remote) - response = transmit_request(connection, zip_data.getvalue()) - handle_response(options, connection, response) - finally: - if connection is not None: - connection.close() Index: lldb/scripts/swig_bot_lib/local.py =================================================================== --- lldb/scripts/swig_bot_lib/local.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python - -""" -Shared functionality used by `client` and `server` when generating or preparing -to generate SWIG on the local machine. -""" - -# Future imports -from __future__ import absolute_import -from __future__ import print_function - -# Python modules -import argparse -import imp -import io -import logging -import os -import subprocess -import sys -import tempfile -import zipfile - -# LLDB modules -import use_lldb_suite - -# Package imports -from lldbsuite.support import fs - - -class LocalConfig(object): - src_root = None - target_dir = None - languages = None - swig_executable = None - - -def pack_archive(bytes_io, src_root, filters): - logging.info("Creating input file package...") - zip_file = None - try: - # It's possible that a custom-built interpreter will not have the - # standard zlib module. If so, we can only store, not compress. By - # try to compress since we usually have a standard Python distribution. - zip_file = zipfile.ZipFile(bytes_io, mode='w', - compression=zipfile.ZIP_DEFLATED) - except RuntimeError: - zip_file = zipfile.ZipFile(bytes_io, mode='w', - compression=zipfile.ZIP_STORED) - archive_entries = [] - if filters is not None: - def filter_func(t): - subfolder = t[0] - ext = t[1] - full_path = os.path.normpath(os.path.join(src_root, subfolder)) - candidates = [os.path.normpath(os.path.join(full_path, f)) - for f in os.listdir(full_path)] - actual = [f for f in candidates if os.path.isfile(f) and os.path.splitext(f)[1] == ext] - return (subfolder, [os.path.basename(f) for f in actual]) - archive_entries = map(filter_func, filters) - else: - for (root, dirs, files) in os.walk(src_root): - logging.debug("Adding files {} from directory {} to output package" - .format(files, root)) - if len(files) > 0: - rel_root = os.path.relpath(root, src_root) - archive_entries.append((rel_root, files)) - - archive_entries = list(archive_entries) - for entry in archive_entries: - subfolder = entry[0] - files = list(entry[1]) - for file in files: - rel_path = os.path.normpath(os.path.join(subfolder, file)) - full_path = os.path.join(src_root, rel_path) - logging.info("{} -> {}".format(full_path, rel_path)) - zip_file.write(full_path, rel_path) - - return zip_file - - -def unpack_archive(folder, archive_bytes): - zip_data = io.BytesIO(archive_bytes) - logging.debug("Opening zip archive...") - zip_file = zipfile.ZipFile(zip_data, mode='r') - zip_file.extractall(folder) - zip_file.close() - - -def generate(options): - include_folder = os.path.join(options.src_root, "include") - in_file = os.path.join(options.src_root, "scripts", "lldb.swig") - include_folder = os.path.normcase(include_folder) - - for lang in options.languages: - lang = lang.lower() - out_dir = os.path.join(options.target_dir, lang.title()) - if not os.path.exists(out_dir): - os.makedirs(out_dir) - out_file = os.path.join(out_dir, "LLDBWrap{}.cpp".format(lang.title())) - swig_command = [ - options.swig_executable, - "-c++", - ] - swig_command.append("-" + lang) - if lang == "python": - swig_command.append("-threads") - - swig_command.extend([ - "-I" + include_folder, - "-D__STDC_LIMIT_MACROS", - "-D__STDC_CONSTANT_MACROS", - "-outdir", out_dir, - "-o", out_file, - in_file - ]) - - logging.info("generating swig {} bindings into {}" - .format(lang, out_dir)) - logging.debug("swig command line: {}".format(swig_command)) - try: - # Execute swig - swig_output = subprocess.check_output( - swig_command, stderr=subprocess.STDOUT, universal_newlines=True) - - logging.info("swig generation succeeded") - if swig_output is not None and len(swig_output) > 0: - logging.info("swig output: %s", swig_output) - return (0, swig_output) - except subprocess.CalledProcessError as e: - logging.error("An error occurred executing swig. returncode={}" - .format(e.returncode)) - logging.error(e.output) - return (e.returncode, e.output) Index: lldb/scripts/swig_bot_lib/remote.py =================================================================== --- lldb/scripts/swig_bot_lib/remote.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -""" -Shared functionality used by `client` and `server` when dealing with -remote transmission -""" - -# Future imports -from __future__ import absolute_import -from __future__ import print_function - -# Python modules -import json -import logging -import os -import socket -import struct -import sys - -# LLDB modules -import use_lldb_suite - - -def generate_config(languages): - config = {"languages": languages} - return json.dumps(config) - - -def parse_config(json_reader): - json_data = json_reader.read() - options_dict = json.loads(json_data) - return options_dict - - -def serialize_response_status(status): - status = {"retcode": status[0], "output": status[1]} - return json.dumps(status) - - -def deserialize_response_status(json_reader): - json_data = json_reader.read() - response_dict = json.loads(json_data) - return (response_dict["retcode"], response_dict["output"]) Index: lldb/scripts/swig_bot_lib/server.py =================================================================== --- lldb/scripts/swig_bot_lib/server.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python - -""" -SWIG generation server. Listens for connections from swig generation clients -and runs swig in the requested fashion, sending back the results. -""" - -# Future imports -from __future__ import absolute_import -from __future__ import print_function - -# Python modules -import argparse -import io -import logging -import os -import select -import shutil -import socket -import struct -import sys -import tempfile -import traceback - -# LLDB modules -import use_lldb_suite -from lldbsuite.support import fs -from lldbsuite.support import sockutil - -# package imports -from . import local -from . import remote - -default_port = 8537 - - -def add_subparser_args(parser): - parser.add_argument( - "--port", - action="store", - default=default_port, - help=("The local port to bind to")) - - parser.add_argument( - "--swig-executable", - action="store", - default=fs.find_executable("swig"), - dest="swig_executable") - - -def finalize_subparser_options(options): - pass - - -def initialize_listening_socket(options): - logging.debug("Creating socket...") - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - logging.info("Binding to ip address '', port {}".format(options.port)) - s.bind(('', options.port)) - - logging.debug("Putting socket in listen mode...") - s.listen() - return s - - -def accept_once(sock, options): - logging.debug("Waiting for connection...") - while True: - rlist, wlist, xlist = select.select([sock], [], [], 0.5) - if not rlist: - continue - - client, addr = sock.accept() - logging.info("Received connection from {}".format(addr)) - data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0] - logging.debug("Expecting {} bytes of data from client" - .format(data_size)) - data = sockutil.recvall(client, data_size) - logging.info("Received {} bytes of data from client" - .format(len(data))) - - pack_location = None - try: - tempfolder = os.path.join(tempfile.gettempdir(), "swig-bot") - os.makedirs(tempfolder, exist_ok=True) - - pack_location = tempfile.mkdtemp(dir=tempfolder) - logging.debug("Extracting archive to {}".format(pack_location)) - - local.unpack_archive(pack_location, data) - logging.debug("Successfully unpacked archive...") - - config_file = os.path.normpath(os.path.join(pack_location, - "config.json")) - parsed_config = remote.parse_config(io.open(config_file)) - config = local.LocalConfig() - config.languages = parsed_config["languages"] - config.swig_executable = options.swig_executable - config.src_root = pack_location - config.target_dir = os.path.normpath( - os.path.join(config.src_root, "output")) - logging.info( - "Running swig. languages={}, swig={}, src_root={}, target={}" - .format(config.languages, config.swig_executable, - config.src_root, config.target_dir)) - - status = local.generate(config) - logging.debug("Finished running swig. Packaging up files {}" - .format(os.listdir(config.target_dir))) - zip_data = io.BytesIO() - zip_file = local.pack_archive(zip_data, config.target_dir, None) - response_status = remote.serialize_response_status(status) - logging.debug("Sending response status {}".format(response_status)) - logging.info("(swig output) -> swig_output.json") - zip_file.writestr("swig_output.json", response_status) - - zip_file.close() - response_data = zip_data.getvalue() - logging.info("Sending {} byte response".format(len(response_data))) - client.sendall(struct.pack("!I", len(response_data))) - client.sendall(response_data) - finally: - if pack_location is not None: - logging.debug("Removing temporary folder {}" - .format(pack_location)) - shutil.rmtree(pack_location) - - -def accept_loop(sock, options): - while True: - try: - accept_once(sock, options) - except Exception as e: - error = traceback.format_exc() - logging.error("An error occurred while processing the connection.") - logging.error(error) - - -def run(options): - print(options) - sock = initialize_listening_socket(options) - accept_loop(sock, options) - return options