Index: tools/scan-build-py/bin/analyze-c++ =================================================================== --- tools/scan-build-py/bin/analyze-c++ +++ tools/scan-build-py/bin/analyze-c++ @@ -11,4 +11,4 @@ sys.path.append(os.path.dirname(this_dir)) from libscanbuild.analyze import analyze_build_wrapper -sys.exit(analyze_build_wrapper(True)) +sys.exit(analyze_build_wrapper()) Index: tools/scan-build-py/bin/analyze-cc =================================================================== --- tools/scan-build-py/bin/analyze-cc +++ tools/scan-build-py/bin/analyze-cc @@ -11,4 +11,4 @@ sys.path.append(os.path.dirname(this_dir)) from libscanbuild.analyze import analyze_build_wrapper -sys.exit(analyze_build_wrapper(False)) +sys.exit(analyze_build_wrapper()) Index: tools/scan-build-py/bin/intercept-c++ =================================================================== --- tools/scan-build-py/bin/intercept-c++ +++ tools/scan-build-py/bin/intercept-c++ @@ -11,4 +11,4 @@ sys.path.append(os.path.dirname(this_dir)) from libscanbuild.intercept import intercept_build_wrapper -sys.exit(intercept_build_wrapper(True)) +sys.exit(intercept_build_wrapper()) Index: tools/scan-build-py/bin/intercept-cc =================================================================== --- tools/scan-build-py/bin/intercept-cc +++ tools/scan-build-py/bin/intercept-cc @@ -11,4 +11,4 @@ sys.path.append(os.path.dirname(this_dir)) from libscanbuild.intercept import intercept_build_wrapper -sys.exit(intercept_build_wrapper(False)) +sys.exit(intercept_build_wrapper()) Index: tools/scan-build-py/libscanbuild/__init__.py =================================================================== --- tools/scan-build-py/libscanbuild/__init__.py +++ tools/scan-build-py/libscanbuild/__init__.py @@ -4,13 +4,21 @@ # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. """ This module is a collection of methods commonly used in this project. """ +import collections import functools +import json import logging import os import os.path +import re +import shlex import subprocess import sys +ENVIRONMENT_KEY = 'INTERCEPT_BUILD' + +Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd']) + def duplicate_check(method): """ Predicate to detect duplicated entries. @@ -75,31 +83,53 @@ raise ex -def initialize_logging(verbose_level): - """ Output content controlled by the verbosity level. """ +def reconfigure_logging(verbose_level): + """ Reconfigure logging level and format based on the verbose flag. - level = logging.WARNING - min(logging.WARNING, (10 * verbose_level)) + :param verbose_level: number of `-v` flags received by the command + :return: no return value + """ + # Exit when nothing to do. + if verbose_level == 0: + return + root = logging.getLogger() + # Tune logging level. + level = logging.WARNING - min(logging.WARNING, (10 * verbose_level)) + root.setLevel(level) + # Be verbose with messages. if verbose_level <= 3: - fmt_string = '{0}: %(levelname)s: %(message)s' + fmt_string = '%(name)s: %(levelname)s: %(message)s' else: - fmt_string = '{0}: %(levelname)s: %(funcName)s: %(message)s' - - program = os.path.basename(sys.argv[0]) - logging.basicConfig(format=fmt_string.format(program), level=level) + fmt_string = '%(name)s: %(levelname)s: %(funcName)s: %(message)s' + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter(fmt=fmt_string)) + root.handlers = [handler] def command_entry_point(function): - """ Decorator for command entry points. """ + """ Decorator for command entry methods. + + The decorator initialize/shutdown logging and guard on programming + errors (catch exceptions). + + The decorated method can have arbitrary parameters, the return value will + be the exit code of the process. """ @functools.wraps(function) def wrapper(*args, **kwargs): + """ Do housekeeping tasks and execute the wrapped method. """ - exit_code = 127 try: - exit_code = function(*args, **kwargs) + logging.basicConfig(format='%(name)s: %(message)s', + level=logging.WARNING, + stream=sys.stdout) + # This hack to get the executable name as %(name). + logging.getLogger().name = os.path.basename(sys.argv[0]) + return function(*args, **kwargs) except KeyboardInterrupt: - logging.warning('Keyboard interupt') + logging.warning('Keyboard interrupt') + return 130 # Signal received exit code for bash. except Exception: logging.exception('Internal error.') if logging.getLogger().isEnabledFor(logging.DEBUG): @@ -107,8 +137,79 @@ "to the bug report") else: logging.error("Please run this command again and turn on " - "verbose mode (add '-vvv' as argument).") + "verbose mode (add '-vvvv' as argument).") + return 64 # Some non used exit code for internal errors. finally: - return exit_code + logging.shutdown() return wrapper + + +def wrapper_entry_point(function): + """ Decorator for wrapper command entry methods. + + The decorator itself execute the real compiler call. Then it calls the + decorated method. The method will receive dictionary of parameters. + + - execution: the command executed by the wrapper. + - result: the exit code of the compilation. + + The return value will be the exit code of the compiler call. (The + decorated method return value is ignored.) + + If the decorated method throws exception, it will be caught and logged. """ + + def is_cxx_compiler(): + """ Find out was it a C++ compiler call. Compiler wrapper names + contain the compiler type. C++ compiler wrappers ends with `c++`, + but might have `.exe` extension on windows. """ + + wrapper_command = os.path.basename(sys.argv[0]) + return re.match(r'(.+)c\+\+(.*)', wrapper_command) + + def run_compiler(executable): + """ Execute compilation with the real compiler. """ + + command = executable + sys.argv[1:] + logging.debug('compilation: %s', command) + result = subprocess.call(command) + logging.debug('compilation exit code: %d', result) + return result + + @functools.wraps(function) + def wrapper(): + """ It executes the compilation and calls the wrapped method. """ + + # Get relevant parameters from environment. + parameters = json.loads(os.environ[ENVIRONMENT_KEY]) + reconfigure_logging(parameters['verbose']) + # Execute the requested compilation. Do crash if anything goes wrong. + cxx = is_cxx_compiler() + compiler = parameters['cxx'] if cxx else parameters['cc'] + result = run_compiler(compiler) + # Call the wrapped method and ignore it's return value. + try: + call = Execution( + pid=os.getpid(), + cwd=os.getcwd(), + cmd=['c++' if cxx else 'cc'] + sys.argv[1:]) + function(execution=call, result=result) + except: + logging.exception('Compiler wrapper failed complete.') + finally: + # Always return the real compiler exit code. + return result + + return wrapper + + +def wrapper_environment(args): + """ Set up environment for interpose compiler wrapper.""" + + return { + ENVIRONMENT_KEY: json.dumps({ + 'verbose': args.verbose, + 'cc': shlex.split(args.cc), + 'cxx': shlex.split(args.cxx) + }) + } Index: tools/scan-build-py/libscanbuild/analyze.py =================================================================== --- tools/scan-build-py/libscanbuild/analyze.py +++ tools/scan-build-py/libscanbuild/analyze.py @@ -23,8 +23,8 @@ import multiprocessing import contextlib import datetime -from libscanbuild import initialize_logging, tempdir, command_entry_point, \ - run_build +from libscanbuild import command_entry_point, wrapper_entry_point, \ + wrapper_environment, reconfigure_logging, run_build, tempdir from libscanbuild.runner import run from libscanbuild.intercept import capture from libscanbuild.report import document @@ -46,8 +46,9 @@ validate(parser, args, from_build_command) # setup logging - initialize_logging(args.verbose) - logging.debug('Parsed arguments: %s', args) + reconfigure_logging(args.verbose) + logging.debug('Raw arguments %s', sys.argv) + with report_directory(args.output, args.keep_empty) as target_dir: if not from_build_command: @@ -130,13 +131,11 @@ """ Set up environment for build command to interpose compiler wrapper. """ environment = dict(os.environ) + environment.update(wrapper_environment(args)) environment.update({ 'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC), 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX), - 'ANALYZE_BUILD_CC': args.cc, - 'ANALYZE_BUILD_CXX': args.cxx, 'ANALYZE_BUILD_CLANG': args.clang if need_analyzer(args.build) else '', - 'ANALYZE_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'WARNING', 'ANALYZE_BUILD_REPORT_DIR': destination, 'ANALYZE_BUILD_REPORT_FORMAT': args.output_format, 'ANALYZE_BUILD_REPORT_FAILURES': 'yes' if args.output_failures else '', @@ -146,51 +145,41 @@ return environment -def analyze_build_wrapper(cplusplus): +@command_entry_point +@wrapper_entry_point +def analyze_build_wrapper(**kwargs): """ Entry point for `analyze-cc` and `analyze-c++` compiler wrappers. """ - # initialize wrapper logging - logging.basicConfig(format='analyze: %(levelname)s: %(message)s', - level=os.getenv('ANALYZE_BUILD_VERBOSE', 'INFO')) - # execute with real compiler - compiler = os.getenv('ANALYZE_BUILD_CXX', 'c++') if cplusplus \ - else os.getenv('ANALYZE_BUILD_CC', 'cc') - compilation = [compiler] + sys.argv[1:] - logging.info('execute compiler: %s', compilation) - result = subprocess.call(compilation) - # exit when it fails, ... - if result or not os.getenv('ANALYZE_BUILD_CLANG'): - return result - # ... and run the analyzer if all went well. - try: - # check is it a compilation - compilation = split_command(sys.argv) - if compilation is None: - return result - # collect the needed parameters from environment, crash when missing - parameters = { - 'clang': os.getenv('ANALYZE_BUILD_CLANG'), - 'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'), - 'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'), - 'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'), - 'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS', - '').split(' '), - 'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'), - 'directory': os.getcwd(), - 'command': [sys.argv[0], '-c'] + compilation.flags - } - # call static analyzer against the compilation - for source in compilation.files: - parameters.update({'file': source}) - logging.debug('analyzer parameters %s', parameters) - current = run(parameters) - # display error message from the static analyzer - if current is not None: - for line in current['error_output']: - logging.info(line.rstrip()) - except Exception: - logging.exception("run analyzer inside compiler wrapper failed.") - return result + # don't run analyzer when compilation fails. or when it's not requested. + if kwargs['result'] or not os.getenv('ANALYZE_BUILD_CLANG'): + return + + execution = kwargs['execution'] + # check is it a compilation? + compilation = split_command(execution.cmd) + if compilation is None: + return + # collect the needed parameters from environment, crash when missing + parameters = { + 'clang': os.getenv('ANALYZE_BUILD_CLANG'), + 'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'), + 'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'), + 'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'), + 'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS', + '').split(' '), + 'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'), + 'directory': execution.cwd, + 'command': [execution.cmd[0], '-c'] + compilation.flags + } + # call static analyzer against the compilation + for source in compilation.files: + parameters.update({'file': source}) + logging.debug('analyzer parameters %s', parameters) + current = run(parameters) + # display error message from the static analyzer + if current is not None: + for line in current['error_output']: + logging.info(line.rstrip()) @contextlib.contextmanager Index: tools/scan-build-py/libscanbuild/intercept.py =================================================================== --- tools/scan-build-py/libscanbuild/intercept.py +++ tools/scan-build-py/libscanbuild/intercept.py @@ -31,8 +31,9 @@ import logging import subprocess from libear import build_libear, TemporaryDirectory -from libscanbuild import command_entry_point, run_build, run_command -from libscanbuild import duplicate_check, tempdir, initialize_logging +from libscanbuild import command_entry_point, wrapper_entry_point, \ + run_command, run_build, wrapper_environment, reconfigure_logging +from libscanbuild import duplicate_check, tempdir from libscanbuild.compilation import split_command from libscanbuild.shell import encode, decode @@ -44,6 +45,7 @@ COMPILER_WRAPPER_CC = 'intercept-cc' COMPILER_WRAPPER_CXX = 'intercept-c++' +TRACE_FILE_EXTENSION = '.cmd' # same as in ear.c WRAPPER_ONLY_PLATFORMS = frozenset({'win32', 'cygwin'}) @@ -54,8 +56,8 @@ parser = create_parser() args = parser.parse_args() - initialize_logging(args.verbose) - logging.debug('Parsed arguments: %s', args) + reconfigure_logging(args.verbose) + logging.debug('Raw arguments %s', sys.argv) if not args.build: parser.print_help() @@ -126,12 +128,10 @@ if not libear_path: logging.debug('intercept gonna use compiler wrappers') + environment.update(wrapper_environment(args)) environment.update({ 'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC), - 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX), - 'INTERCEPT_BUILD_CC': c_compiler, - 'INTERCEPT_BUILD_CXX': cxx_compiler, - 'INTERCEPT_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'INFO' + 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX) }) elif sys.platform == 'darwin': logging.debug('intercept gonna preload libear on OSX') @@ -146,42 +146,44 @@ return environment -def intercept_build_wrapper(cplusplus): +@command_entry_point +@wrapper_entry_point +def intercept_build_wrapper(**kwargs): """ Entry point for `intercept-cc` and `intercept-c++` compiler wrappers. - It does generate execution report into target directory. And execute - the wrapped compilation with the real compiler. The parameters for - report and execution are from environment variables. + It does generate execution report into target directory. + The target directory name is from environment variables. """ - Those parameters which for 'libear' library can't have meaningful - values are faked. """ + message_prefix = 'execution report might be incomplete: %s' - # initialize wrapper logging - logging.basicConfig(format='intercept: %(levelname)s: %(message)s', - level=os.getenv('INTERCEPT_BUILD_VERBOSE', 'INFO')) - # write report + target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR') + if not target_dir: + logging.warning(message_prefix, 'missing target directory') + return + # write current execution info to the pid file try: - target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR') - if not target_dir: - raise UserWarning('exec report target directory not found') - pid = str(os.getpid()) - target_file = os.path.join(target_dir, pid + '.cmd') - logging.debug('writing exec report to: %s', target_file) - with open(target_file, 'ab') as handler: - working_dir = os.getcwd() - command = US.join(sys.argv) + US - content = RS.join([pid, pid, 'wrapper', working_dir, command]) + GS - handler.write(content.encode('utf-8')) + target_file_name = str(os.getpid()) + TRACE_FILE_EXTENSION + target_file = os.path.join(target_dir, target_file_name) + logging.debug('writing execution report to: %s', target_file) + write_exec_trace(target_file, kwargs['execution']) except IOError: - logging.exception('writing exec report failed') - except UserWarning as warning: - logging.warning(warning) - # execute with real compiler - compiler = os.getenv('INTERCEPT_BUILD_CXX', 'c++') if cplusplus \ - else os.getenv('INTERCEPT_BUILD_CC', 'cc') - compilation = [compiler] + sys.argv[1:] - logging.debug('execute compiler: %s', compilation) - return subprocess.call(compilation) + logging.warning(message_prefix, 'io problem') + + +def write_exec_trace(filename, entry): + """ Write execution report file. + + This method shall be sync with the execution report writer in interception + library. The entry in the file is a JSON objects. + + :param filename: path to the output execution trace file, + :param entry: the Execution object to append to that file. """ + + with open(filename, 'ab') as handler: + pid = str(entry.pid) + command = US.join(entry.cmd) + US + content = RS.join([pid, pid, 'wrapper', entry.cwd, command]) + GS + handler.write(content.encode('utf-8')) def parse_exec_trace(filename):