Index: cfe/trunk/tools/scan-build-py/bin/analyze-c++ =================================================================== --- cfe/trunk/tools/scan-build-py/bin/analyze-c++ +++ cfe/trunk/tools/scan-build-py/bin/analyze-c++ @@ -10,5 +10,5 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.dirname(this_dir)) -from libscanbuild.analyze import analyze_build_wrapper -sys.exit(analyze_build_wrapper(True)) +from libscanbuild.analyze import analyze_compiler_wrapper +sys.exit(analyze_compiler_wrapper()) Index: cfe/trunk/tools/scan-build-py/bin/analyze-cc =================================================================== --- cfe/trunk/tools/scan-build-py/bin/analyze-cc +++ cfe/trunk/tools/scan-build-py/bin/analyze-cc @@ -10,5 +10,5 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.dirname(this_dir)) -from libscanbuild.analyze import analyze_build_wrapper -sys.exit(analyze_build_wrapper(False)) +from libscanbuild.analyze import analyze_compiler_wrapper +sys.exit(analyze_compiler_wrapper()) Index: cfe/trunk/tools/scan-build-py/bin/intercept-c++ =================================================================== --- cfe/trunk/tools/scan-build-py/bin/intercept-c++ +++ cfe/trunk/tools/scan-build-py/bin/intercept-c++ @@ -10,5 +10,5 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.dirname(this_dir)) -from libscanbuild.intercept import intercept_build_wrapper -sys.exit(intercept_build_wrapper(True)) +from libscanbuild.intercept import intercept_compiler_wrapper +sys.exit(intercept_compiler_wrapper()) Index: cfe/trunk/tools/scan-build-py/bin/intercept-cc =================================================================== --- cfe/trunk/tools/scan-build-py/bin/intercept-cc +++ cfe/trunk/tools/scan-build-py/bin/intercept-cc @@ -10,5 +10,5 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.dirname(this_dir)) -from libscanbuild.intercept import intercept_build_wrapper -sys.exit(intercept_build_wrapper(False)) +from libscanbuild.intercept import intercept_compiler_wrapper +sys.exit(intercept_compiler_wrapper()) Index: cfe/trunk/tools/scan-build-py/libscanbuild/__init__.py =================================================================== --- cfe/trunk/tools/scan-build-py/libscanbuild/__init__.py +++ cfe/trunk/tools/scan-build-py/libscanbuild/__init__.py @@ -4,13 +4,21 @@ # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. """ This module is a collection of methods commonly used in this project. """ +import collections import functools +import json import logging import os import os.path +import re +import shlex import subprocess import sys +ENVIRONMENT_KEY = 'INTERCEPT_BUILD' + +Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd']) + def duplicate_check(method): """ Predicate to detect duplicated entries. @@ -75,31 +83,53 @@ raise ex -def initialize_logging(verbose_level): - """ Output content controlled by the verbosity level. """ +def reconfigure_logging(verbose_level): + """ Reconfigure logging level and format based on the verbose flag. - level = logging.WARNING - min(logging.WARNING, (10 * verbose_level)) + :param verbose_level: number of `-v` flags received by the command + :return: no return value + """ + # Exit when nothing to do. + if verbose_level == 0: + return + root = logging.getLogger() + # Tune logging level. + level = logging.WARNING - min(logging.WARNING, (10 * verbose_level)) + root.setLevel(level) + # Be verbose with messages. if verbose_level <= 3: - fmt_string = '{0}: %(levelname)s: %(message)s' + fmt_string = '%(name)s: %(levelname)s: %(message)s' else: - fmt_string = '{0}: %(levelname)s: %(funcName)s: %(message)s' - - program = os.path.basename(sys.argv[0]) - logging.basicConfig(format=fmt_string.format(program), level=level) + fmt_string = '%(name)s: %(levelname)s: %(funcName)s: %(message)s' + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter(fmt=fmt_string)) + root.handlers = [handler] def command_entry_point(function): - """ Decorator for command entry points. """ + """ Decorator for command entry methods. + + The decorator initialize/shutdown logging and guard on programming + errors (catch exceptions). + + The decorated method can have arbitrary parameters, the return value will + be the exit code of the process. """ @functools.wraps(function) def wrapper(*args, **kwargs): + """ Do housekeeping tasks and execute the wrapped method. """ - exit_code = 127 try: - exit_code = function(*args, **kwargs) + logging.basicConfig(format='%(name)s: %(message)s', + level=logging.WARNING, + stream=sys.stdout) + # This hack to get the executable name as %(name). + logging.getLogger().name = os.path.basename(sys.argv[0]) + return function(*args, **kwargs) except KeyboardInterrupt: - logging.warning('Keyboard interupt') + logging.warning('Keyboard interrupt') + return 130 # Signal received exit code for bash. except Exception: logging.exception('Internal error.') if logging.getLogger().isEnabledFor(logging.DEBUG): @@ -107,8 +137,75 @@ "to the bug report") else: logging.error("Please run this command again and turn on " - "verbose mode (add '-vvv' as argument).") + "verbose mode (add '-vvvv' as argument).") + return 64 # Some non used exit code for internal errors. finally: - return exit_code + logging.shutdown() return wrapper + + +def compiler_wrapper(function): + """ Implements compiler wrapper base functionality. + + A compiler wrapper executes the real compiler, then implement some + functionality, then returns with the real compiler exit code. + + :param function: the extra functionality what the wrapper want to + do on top of the compiler call. If it throws exception, it will be + caught and logged. + :return: the exit code of the real compiler. + + The :param function: will receive the following arguments: + + :param result: the exit code of the compilation. + :param execution: the command executed by the wrapper. """ + + def is_cxx_compiler(): + """ Find out was it a C++ compiler call. Compiler wrapper names + contain the compiler type. C++ compiler wrappers ends with `c++`, + but might have `.exe` extension on windows. """ + + wrapper_command = os.path.basename(sys.argv[0]) + return re.match(r'(.+)c\+\+(.*)', wrapper_command) + + def run_compiler(executable): + """ Execute compilation with the real compiler. """ + + command = executable + sys.argv[1:] + logging.debug('compilation: %s', command) + result = subprocess.call(command) + logging.debug('compilation exit code: %d', result) + return result + + # Get relevant parameters from environment. + parameters = json.loads(os.environ[ENVIRONMENT_KEY]) + reconfigure_logging(parameters['verbose']) + # Execute the requested compilation. Do crash if anything goes wrong. + cxx = is_cxx_compiler() + compiler = parameters['cxx'] if cxx else parameters['cc'] + result = run_compiler(compiler) + # Call the wrapped method and ignore it's return value. + try: + call = Execution( + pid=os.getpid(), + cwd=os.getcwd(), + cmd=['c++' if cxx else 'cc'] + sys.argv[1:]) + function(result, call) + except: + logging.exception('Compiler wrapper failed complete.') + finally: + # Always return the real compiler exit code. + return result + + +def wrapper_environment(args): + """ Set up environment for interpose compiler wrapper.""" + + return { + ENVIRONMENT_KEY: json.dumps({ + 'verbose': args.verbose, + 'cc': shlex.split(args.cc), + 'cxx': shlex.split(args.cxx) + }) + } Index: cfe/trunk/tools/scan-build-py/libscanbuild/analyze.py =================================================================== --- cfe/trunk/tools/scan-build-py/libscanbuild/analyze.py +++ cfe/trunk/tools/scan-build-py/libscanbuild/analyze.py @@ -19,19 +19,18 @@ import argparse import logging import tempfile -import subprocess import multiprocessing import contextlib import datetime -from libscanbuild import initialize_logging, tempdir, command_entry_point, \ - run_build +from libscanbuild import command_entry_point, compiler_wrapper, \ + wrapper_environment, reconfigure_logging, run_build, tempdir from libscanbuild.runner import run from libscanbuild.intercept import capture from libscanbuild.report import document from libscanbuild.clang import get_checkers from libscanbuild.compilation import split_command -__all__ = ['analyze_build_main', 'analyze_build_wrapper'] +__all__ = ['analyze_build_main', 'analyze_compiler_wrapper'] COMPILER_WRAPPER_CC = 'analyze-cc' COMPILER_WRAPPER_CXX = 'analyze-c++' @@ -46,8 +45,8 @@ validate(parser, args, from_build_command) # setup logging - initialize_logging(args.verbose) - logging.debug('Parsed arguments: %s', args) + reconfigure_logging(args.verbose) + logging.debug('Raw arguments %s', sys.argv) with report_directory(args.output, args.keep_empty) as target_dir: if not from_build_command: @@ -130,13 +129,11 @@ """ Set up environment for build command to interpose compiler wrapper. """ environment = dict(os.environ) + environment.update(wrapper_environment(args)) environment.update({ 'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC), 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX), - 'ANALYZE_BUILD_CC': args.cc, - 'ANALYZE_BUILD_CXX': args.cxx, 'ANALYZE_BUILD_CLANG': args.clang if need_analyzer(args.build) else '', - 'ANALYZE_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'WARNING', 'ANALYZE_BUILD_REPORT_DIR': destination, 'ANALYZE_BUILD_REPORT_FORMAT': args.output_format, 'ANALYZE_BUILD_REPORT_FAILURES': 'yes' if args.output_failures else '', @@ -146,51 +143,45 @@ return environment -def analyze_build_wrapper(cplusplus): +@command_entry_point +def analyze_compiler_wrapper(): """ Entry point for `analyze-cc` and `analyze-c++` compiler wrappers. """ - # initialize wrapper logging - logging.basicConfig(format='analyze: %(levelname)s: %(message)s', - level=os.getenv('ANALYZE_BUILD_VERBOSE', 'INFO')) - # execute with real compiler - compiler = os.getenv('ANALYZE_BUILD_CXX', 'c++') if cplusplus \ - else os.getenv('ANALYZE_BUILD_CC', 'cc') - compilation = [compiler] + sys.argv[1:] - logging.info('execute compiler: %s', compilation) - result = subprocess.call(compilation) - # exit when it fails, ... + return compiler_wrapper(analyze_compiler_wrapper_impl) + + +def analyze_compiler_wrapper_impl(result, execution): + """ Implements analyzer compiler wrapper functionality. """ + + # don't run analyzer when compilation fails. or when it's not requested. if result or not os.getenv('ANALYZE_BUILD_CLANG'): - return result - # ... and run the analyzer if all went well. - try: - # check is it a compilation - compilation = split_command(sys.argv) - if compilation is None: - return result - # collect the needed parameters from environment, crash when missing - parameters = { - 'clang': os.getenv('ANALYZE_BUILD_CLANG'), - 'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'), - 'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'), - 'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'), - 'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS', - '').split(' '), - 'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'), - 'directory': os.getcwd(), - 'command': [sys.argv[0], '-c'] + compilation.flags - } - # call static analyzer against the compilation - for source in compilation.files: - parameters.update({'file': source}) - logging.debug('analyzer parameters %s', parameters) - current = run(parameters) - # display error message from the static analyzer - if current is not None: - for line in current['error_output']: - logging.info(line.rstrip()) - except Exception: - logging.exception("run analyzer inside compiler wrapper failed.") - return result + return + + # check is it a compilation? + compilation = split_command(execution.cmd) + if compilation is None: + return + # collect the needed parameters from environment, crash when missing + parameters = { + 'clang': os.getenv('ANALYZE_BUILD_CLANG'), + 'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'), + 'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'), + 'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'), + 'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS', + '').split(' '), + 'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'), + 'directory': execution.cwd, + 'command': [execution.cmd[0], '-c'] + compilation.flags + } + # call static analyzer against the compilation + for source in compilation.files: + parameters.update({'file': source}) + logging.debug('analyzer parameters %s', parameters) + current = run(parameters) + # display error message from the static analyzer + if current is not None: + for line in current['error_output']: + logging.info(line.rstrip()) @contextlib.contextmanager Index: cfe/trunk/tools/scan-build-py/libscanbuild/intercept.py =================================================================== --- cfe/trunk/tools/scan-build-py/libscanbuild/intercept.py +++ cfe/trunk/tools/scan-build-py/libscanbuild/intercept.py @@ -29,14 +29,14 @@ import glob import argparse import logging -import subprocess from libear import build_libear, TemporaryDirectory -from libscanbuild import command_entry_point, run_build, run_command -from libscanbuild import duplicate_check, tempdir, initialize_logging +from libscanbuild import command_entry_point, compiler_wrapper, \ + wrapper_environment, run_command, run_build, reconfigure_logging +from libscanbuild import duplicate_check, tempdir from libscanbuild.compilation import split_command from libscanbuild.shell import encode, decode -__all__ = ['capture', 'intercept_build_main', 'intercept_build_wrapper'] +__all__ = ['capture', 'intercept_build_main', 'intercept_compiler_wrapper'] GS = chr(0x1d) RS = chr(0x1e) @@ -44,6 +44,7 @@ COMPILER_WRAPPER_CC = 'intercept-cc' COMPILER_WRAPPER_CXX = 'intercept-c++' +TRACE_FILE_EXTENSION = '.cmd' # same as in ear.c WRAPPER_ONLY_PLATFORMS = frozenset({'win32', 'cygwin'}) @@ -54,8 +55,8 @@ parser = create_parser() args = parser.parse_args() - initialize_logging(args.verbose) - logging.debug('Parsed arguments: %s', args) + reconfigure_logging(args.verbose) + logging.debug('Raw arguments %s', sys.argv) if not args.build: parser.print_help() @@ -126,12 +127,10 @@ if not libear_path: logging.debug('intercept gonna use compiler wrappers') + environment.update(wrapper_environment(args)) environment.update({ 'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC), - 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX), - 'INTERCEPT_BUILD_CC': c_compiler, - 'INTERCEPT_BUILD_CXX': cxx_compiler, - 'INTERCEPT_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'INFO' + 'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX) }) elif sys.platform == 'darwin': logging.debug('intercept gonna preload libear on OSX') @@ -146,42 +145,49 @@ return environment -def intercept_build_wrapper(cplusplus): - """ Entry point for `intercept-cc` and `intercept-c++` compiler wrappers. +@command_entry_point +def intercept_compiler_wrapper(): + """ Entry point for `intercept-cc` and `intercept-c++`. """ + + return compiler_wrapper(intercept_compiler_wrapper_impl) + + +def intercept_compiler_wrapper_impl(_, execution): + """ Implement intercept compiler wrapper functionality. + + It does generate execution report into target directory. + The target directory name is from environment variables. """ + + message_prefix = 'execution report might be incomplete: %s' - It does generate execution report into target directory. And execute - the wrapped compilation with the real compiler. The parameters for - report and execution are from environment variables. - - Those parameters which for 'libear' library can't have meaningful - values are faked. """ - - # initialize wrapper logging - logging.basicConfig(format='intercept: %(levelname)s: %(message)s', - level=os.getenv('INTERCEPT_BUILD_VERBOSE', 'INFO')) - # write report + target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR') + if not target_dir: + logging.warning(message_prefix, 'missing target directory') + return + # write current execution info to the pid file try: - target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR') - if not target_dir: - raise UserWarning('exec report target directory not found') - pid = str(os.getpid()) - target_file = os.path.join(target_dir, pid + '.cmd') - logging.debug('writing exec report to: %s', target_file) - with open(target_file, 'ab') as handler: - working_dir = os.getcwd() - command = US.join(sys.argv) + US - content = RS.join([pid, pid, 'wrapper', working_dir, command]) + GS - handler.write(content.encode('utf-8')) + target_file_name = str(os.getpid()) + TRACE_FILE_EXTENSION + target_file = os.path.join(target_dir, target_file_name) + logging.debug('writing execution report to: %s', target_file) + write_exec_trace(target_file, execution) except IOError: - logging.exception('writing exec report failed') - except UserWarning as warning: - logging.warning(warning) - # execute with real compiler - compiler = os.getenv('INTERCEPT_BUILD_CXX', 'c++') if cplusplus \ - else os.getenv('INTERCEPT_BUILD_CC', 'cc') - compilation = [compiler] + sys.argv[1:] - logging.debug('execute compiler: %s', compilation) - return subprocess.call(compilation) + logging.warning(message_prefix, 'io problem') + + +def write_exec_trace(filename, entry): + """ Write execution report file. + + This method shall be sync with the execution report writer in interception + library. The entry in the file is a JSON objects. + + :param filename: path to the output execution trace file, + :param entry: the Execution object to append to that file. """ + + with open(filename, 'ab') as handler: + pid = str(entry.pid) + command = US.join(entry.cmd) + US + content = RS.join([pid, pid, 'wrapper', entry.cwd, command]) + GS + handler.write(content.encode('utf-8')) def parse_exec_trace(filename):