This is an archive of the discontinued LLVM Phabricator instance.

Differential D72539

[lldb/Scripts] Remove SWIG bot
ClosedPublic

Authored by JDevlieghere on Jan 10 2020, 1:52 PM.

Details

Reviewers
labath
zturner
Group Reviewers
Restricted Project
Commits
rGf2bbe8ede057: [lldb/Scripts] Remove SWIG bot
Summary

I don't think is used and it hasn't been touched since 2015. I think this can go away.

Diff Detail

Event Timeline

JDevlieghere created this revision.Jan 10 2020, 1:52 PM
Herald added a project: Restricted Project. · View Herald TranscriptJan 10 2020, 1:52 PM
labath accepted this revision.Jan 12 2020, 11:59 PM

yep

This revision is now accepted and ready to land.Jan 12 2020, 11:59 PM
Closed by commit rGf2bbe8ede057: [lldb/Scripts] Remove SWIG bot (authored by JDevlieghere). · Explain WhyJan 13 2020, 9:28 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

Diff 237430

lldb/scripts/swig_bot.py

  • This file was deleted.
#!/usr/bin/env python
"""
SWIG generation top-level script. Supports both local and remote generation
of SWIG bindings for multiple languages.
"""
# Python modules
import argparse
import logging
import sys
import traceback
# LLDB modules
import use_lldb_suite
# swig_bot modules
from swig_bot_lib import client
from swig_bot_lib import server
def process_args(args):
parser = argparse.ArgumentParser(
description='Run swig-bot client or server.')
# Create and populate subparser arguments for when swig_bot is
# run in client or server mode
subparsers = parser.add_subparsers(
help="Pass --help to a sub-command to print detailed usage")
client_parser = subparsers.add_parser("client",
help="Run SWIG generation client")
client.add_subparser_args(client_parser)
client_parser.set_defaults(func=run_client)
server_parser = subparsers.add_parser("server",
help="Run SWIG generation server")
server.add_subparser_args(server_parser)
server_parser.set_defaults(func=run_server)
# Arguments to control logging verbosity.
parser.add_argument(
"--verbose", "-v",
action="store_true",
default=False,
help="Increase logging verbosity level.")
options = parser.parse_args(args)
# Set logging level.
if options.verbose:
log_level = logging.DEBUG
else:
log_level = logging.NOTSET
logging.basicConfig(level=log_level)
logging.info("logging is using level: %d", log_level)
return options
def run_client(options):
logging.info("Running swig_bot in client mode")
client.finalize_subparser_options(options)
client.run(options)
def run_server(options):
logging.info("Running swig_bot in server mode")
server.finalize_subparser_options(options)
server.run(options)
if __name__ == "__main__":
options = process_args(sys.argv[1:])
try:
if options.func is None:
logging.error(
"Unknown mode specified. Expected client or server.")
sys.exit(-1)
else:
options.func(options)
except KeyboardInterrupt as e:
logging.info("Ctrl+C received. Shutting down...")
sys.exit(-1)
except Exception as e:
error = traceback.format_exc()
logging.error("An error occurred running swig-bot.")
logging.error(error)

lldb/scripts/swig_bot_lib/__init__.py

  • This file was deleted.
The contents of this file were not changed.

lldb/scripts/swig_bot_lib/client.py

  • This file was deleted.
#!/usr/bin/env python
"""
SWIG generation client. Supports both local and remote generation of SWIG
bindings for multiple languages.
"""
# Future imports
from __future__ import absolute_import
from __future__ import print_function
# Python modules
import argparse
import io
import logging
import os
import socket
import struct
import sys
# LLDB modules
import use_lldb_suite
from lldbsuite.support import fs
from lldbsuite.support import sockutil
# package imports
from . import local
from . import remote
default_ip = "127.0.0.1"
default_port = 8537
def add_subparser_args(parser):
"""Returns options processed from the provided command line.
@param args the command line to process.
"""
# A custom action used by the --local command line option. It can be
# used with either 0 or 1 argument. If used with 0 arguments, it
# searches for a copy of swig located on the physical machine. If
# used with 1 argument, the argument is the path to a swig executable.
class FindLocalSwigAction(argparse.Action):
def __init__(self, option_strings, dest, **kwargs):
super(FindLocalSwigAction, self).__init__(
option_strings, dest, nargs='?', **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
swig_exe = None
if values is None:
swig_exe = fs.find_executable('swig')
else:
swig_exe = values
setattr(namespace, self.dest, os.path.normpath(swig_exe))
# A custom action used by the --remote command line option. It can be
# used with either 0 or 1 arguments. If used with 0 arguments it chooses
# a default connection string. If used with one argument it is a string
# of the form `ip_address[:port]`. If the port is unspecified, the
# default port is used.
class RemoteIpAction(argparse.Action):
def __init__(self, option_strings, dest, **kwargs):
super(RemoteIpAction, self).__init__(
option_strings, dest, nargs='?', **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
ip_port = None
if values is None:
ip_port = (default_ip, default_port)
else:
result = values.split(':')
if len(result) == 1:
ip_port = (result[0], default_port)
elif len(result) == 2:
ip_port = (result[0], int(result[1]))
else:
raise ValueError("Invalid connection string")
setattr(namespace, self.dest, ip_port)
parser.add_argument(
"--local",
action=FindLocalSwigAction,
dest="swig_executable",
help=(
"Run the copy of swig at the specified location, or search PATH"
"if the location is omitted"))
parser.add_argument(
"--remote",
action=RemoteIpAction,
help=(
"Use the given connection string to connect to a remote "
"generation service"))
parser.add_argument(
"--src-root",
required=True,
help="The root folder of the LLDB source tree.")
parser.add_argument(
"--target-dir",
default=os.getcwd(),
help=(
"Specifies the build dir where the language binding "
"should be placed"))
parser.add_argument(
"--language",
dest="languages",
action="append",
help="Specifies the language to generate bindings for")
def finalize_subparser_options(options):
if options.languages is None:
options.languages = ['python']
if options.remote is None and options.swig_executable is None:
logging.error("Must specify either --local or --remote")
sys.exit(-3)
return options
def establish_remote_connection(ip_port):
logging.debug("Creating socket...")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.info("Connecting to server {} on port {}"
.format(ip_port[0], ip_port[1]))
s.connect(ip_port)
logging.info("Connection established...")
return s
def transmit_request(connection, packed_input):
logging.info("Sending {} bytes of compressed data."
.format(len(packed_input)))
connection.sendall(struct.pack("!I", len(packed_input)))
connection.sendall(packed_input)
logging.info("Awaiting response.")
response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0]
logging.debug("Expecting {} byte response".format(response_len))
response = sockutil.recvall(connection, response_len)
return response
def handle_response(options, connection, response):
logging.debug("Received {} byte response.".format(len(response)))
logging.debug("Creating output directory {}"
.format(options.target_dir))
os.makedirs(options.target_dir, exist_ok=True)
logging.info("Unpacking response archive into {}"
.format(options.target_dir))
local.unpack_archive(options.target_dir, response)
response_file_path = os.path.normpath(
os.path.join(options.target_dir, "swig_output.json"))
if not os.path.isfile(response_file_path):
logging.error("Response file '{}' does not exist."
.format(response_file_path))
return
try:
response = remote.deserialize_response_status(
io.open(response_file_path))
if response[0] != 0:
logging.error("An error occurred during generation. Status={}"
.format(response[0]))
logging.error(response[1])
else:
logging.info("SWIG generation successful.")
if len(response[1]) > 0:
logging.info(response[1])
finally:
os.unlink(response_file_path)
def run(options):
if options.remote is None:
logging.info("swig bot client using local swig installation at '{}'"
.format(options.swig_executable))
if not os.path.isfile(options.swig_executable):
logging.error("Swig executable '{}' does not exist."
.format(options.swig_executable))
config = local.LocalConfig()
config.languages = options.languages
config.src_root = options.src_root
config.target_dir = options.target_dir
config.swig_executable = options.swig_executable
local.generate(config)
else:
logging.info("swig bot client using remote generation with server '{}'"
.format(options.remote))
connection = None
try:
config = remote.generate_config(options.languages)
logging.debug("Generated config json {}".format(config))
inputs = [("include/lldb", ".h"),
("include/lldb/API", ".h"),
("scripts", ".swig"),
("scripts/Python", ".swig"),
("scripts/interface", ".i")]
zip_data = io.BytesIO()
packed_input = local.pack_archive(
zip_data, options.src_root, inputs)
logging.info("(null) -> config.json")
packed_input.writestr("config.json", config)
packed_input.close()
connection = establish_remote_connection(options.remote)
response = transmit_request(connection, zip_data.getvalue())
handle_response(options, connection, response)
finally:
if connection is not None:
connection.close()

lldb/scripts/swig_bot_lib/local.py

  • This file was deleted.
#!/usr/bin/env python
"""
Shared functionality used by `client` and `server` when generating or preparing
to generate SWIG on the local machine.
"""
# Future imports
from __future__ import absolute_import
from __future__ import print_function
# Python modules
import argparse
import imp
import io
import logging
import os
import subprocess
import sys
import tempfile
import zipfile
# LLDB modules
import use_lldb_suite
# Package imports
from lldbsuite.support import fs
class LocalConfig(object):
src_root = None
target_dir = None
languages = None
swig_executable = None
def pack_archive(bytes_io, src_root, filters):
logging.info("Creating input file package...")
zip_file = None
try:
# It's possible that a custom-built interpreter will not have the
# standard zlib module. If so, we can only store, not compress. By
# try to compress since we usually have a standard Python distribution.
zip_file = zipfile.ZipFile(bytes_io, mode='w',
compression=zipfile.ZIP_DEFLATED)
except RuntimeError:
zip_file = zipfile.ZipFile(bytes_io, mode='w',
compression=zipfile.ZIP_STORED)
archive_entries = []
if filters is not None:
def filter_func(t):
subfolder = t[0]
ext = t[1]
full_path = os.path.normpath(os.path.join(src_root, subfolder))
candidates = [os.path.normpath(os.path.join(full_path, f))
for f in os.listdir(full_path)]
actual = [f for f in candidates if os.path.isfile(f) and os.path.splitext(f)[1] == ext]
return (subfolder, [os.path.basename(f) for f in actual])
archive_entries = map(filter_func, filters)
else:
for (root, dirs, files) in os.walk(src_root):
logging.debug("Adding files {} from directory {} to output package"
.format(files, root))
if len(files) > 0:
rel_root = os.path.relpath(root, src_root)
archive_entries.append((rel_root, files))
archive_entries = list(archive_entries)
for entry in archive_entries:
subfolder = entry[0]
files = list(entry[1])
for file in files:
rel_path = os.path.normpath(os.path.join(subfolder, file))
full_path = os.path.join(src_root, rel_path)
logging.info("{} -> {}".format(full_path, rel_path))
zip_file.write(full_path, rel_path)
return zip_file
def unpack_archive(folder, archive_bytes):
zip_data = io.BytesIO(archive_bytes)
logging.debug("Opening zip archive...")
zip_file = zipfile.ZipFile(zip_data, mode='r')
zip_file.extractall(folder)
zip_file.close()
def generate(options):
include_folder = os.path.join(options.src_root, "include")
in_file = os.path.join(options.src_root, "scripts", "lldb.swig")
include_folder = os.path.normcase(include_folder)
for lang in options.languages:
lang = lang.lower()
out_dir = os.path.join(options.target_dir, lang.title())
if not os.path.exists(out_dir):
os.makedirs(out_dir)
out_file = os.path.join(out_dir, "LLDBWrap{}.cpp".format(lang.title()))
swig_command = [
options.swig_executable,
"-c++",
]
swig_command.append("-" + lang)
if lang == "python":
swig_command.append("-threads")
swig_command.extend([
"-I" + include_folder,
"-D__STDC_LIMIT_MACROS",
"-D__STDC_CONSTANT_MACROS",
"-outdir", out_dir,
"-o", out_file,
in_file
])
logging.info("generating swig {} bindings into {}"
.format(lang, out_dir))
logging.debug("swig command line: {}".format(swig_command))
try:
# Execute swig
swig_output = subprocess.check_output(
swig_command, stderr=subprocess.STDOUT, universal_newlines=True)
logging.info("swig generation succeeded")
if swig_output is not None and len(swig_output) > 0:
logging.info("swig output: %s", swig_output)
return (0, swig_output)
except subprocess.CalledProcessError as e:
logging.error("An error occurred executing swig. returncode={}"
.format(e.returncode))
logging.error(e.output)
return (e.returncode, e.output)

lldb/scripts/swig_bot_lib/remote.py

  • This file was deleted.
#!/usr/bin/env python
"""
Shared functionality used by `client` and `server` when dealing with
remote transmission
"""
# Future imports
from __future__ import absolute_import
from __future__ import print_function
# Python modules
import json
import logging
import os
import socket
import struct
import sys
# LLDB modules
import use_lldb_suite
def generate_config(languages):
config = {"languages": languages}
return json.dumps(config)
def parse_config(json_reader):
json_data = json_reader.read()
options_dict = json.loads(json_data)
return options_dict
def serialize_response_status(status):
status = {"retcode": status[0], "output": status[1]}
return json.dumps(status)
def deserialize_response_status(json_reader):
json_data = json_reader.read()
response_dict = json.loads(json_data)
return (response_dict["retcode"], response_dict["output"])

lldb/scripts/swig_bot_lib/server.py

  • This file was deleted.
#!/usr/bin/env python
"""
SWIG generation server. Listens for connections from swig generation clients
and runs swig in the requested fashion, sending back the results.
"""
# Future imports
from __future__ import absolute_import
from __future__ import print_function
# Python modules
import argparse
import io
import logging
import os
import select
import shutil
import socket
import struct
import sys
import tempfile
import traceback
# LLDB modules
import use_lldb_suite
from lldbsuite.support import fs
from lldbsuite.support import sockutil
# package imports
from . import local
from . import remote
default_port = 8537
def add_subparser_args(parser):
parser.add_argument(
"--port",
action="store",
default=default_port,
help=("The local port to bind to"))
parser.add_argument(
"--swig-executable",
action="store",
default=fs.find_executable("swig"),
dest="swig_executable")
def finalize_subparser_options(options):
pass
def initialize_listening_socket(options):
logging.debug("Creating socket...")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.info("Binding to ip address '', port {}".format(options.port))
s.bind(('', options.port))
logging.debug("Putting socket in listen mode...")
s.listen()
return s
def accept_once(sock, options):
logging.debug("Waiting for connection...")
while True:
rlist, wlist, xlist = select.select([sock], [], [], 0.5)
if not rlist:
continue
client, addr = sock.accept()
logging.info("Received connection from {}".format(addr))
data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0]
logging.debug("Expecting {} bytes of data from client"
.format(data_size))
data = sockutil.recvall(client, data_size)
logging.info("Received {} bytes of data from client"
.format(len(data)))
pack_location = None
try:
tempfolder = os.path.join(tempfile.gettempdir(), "swig-bot")
os.makedirs(tempfolder, exist_ok=True)
pack_location = tempfile.mkdtemp(dir=tempfolder)
logging.debug("Extracting archive to {}".format(pack_location))
local.unpack_archive(pack_location, data)
logging.debug("Successfully unpacked archive...")
config_file = os.path.normpath(os.path.join(pack_location,
"config.json"))
parsed_config = remote.parse_config(io.open(config_file))
config = local.LocalConfig()
config.languages = parsed_config["languages"]
config.swig_executable = options.swig_executable
config.src_root = pack_location
config.target_dir = os.path.normpath(
os.path.join(config.src_root, "output"))
logging.info(
"Running swig. languages={}, swig={}, src_root={}, target={}"
.format(config.languages, config.swig_executable,
config.src_root, config.target_dir))
status = local.generate(config)
logging.debug("Finished running swig. Packaging up files {}"
.format(os.listdir(config.target_dir)))
zip_data = io.BytesIO()
zip_file = local.pack_archive(zip_data, config.target_dir, None)
response_status = remote.serialize_response_status(status)
logging.debug("Sending response status {}".format(response_status))
logging.info("(swig output) -> swig_output.json")
zip_file.writestr("swig_output.json", response_status)
zip_file.close()
response_data = zip_data.getvalue()
logging.info("Sending {} byte response".format(len(response_data)))
client.sendall(struct.pack("!I", len(response_data)))
client.sendall(response_data)
finally:
if pack_location is not None:
logging.debug("Removing temporary folder {}"
.format(pack_location))
shutil.rmtree(pack_location)
def accept_loop(sock, options):
while True:
try:
accept_once(sock, options)
except Exception as e:
error = traceback.format_exc()
logging.error("An error occurred while processing the connection.")
logging.error(error)
def run(options):
print(options)
sock = initialize_listening_socket(options)
accept_loop(sock, options)
return options
lldb/scripts/swig_bot_lib/local.py