Index: tools/scan-build-py/libscanbuild/compilation.py
===================================================================
--- tools/scan-build-py/libscanbuild/compilation.py
+++ tools/scan-build-py/libscanbuild/compilation.py
@@ -8,11 +8,15 @@
import re
import os
import collections
+import logging
+import json
+from libscanbuild import Execution
+from libscanbuild.shell import decode
__all__ = ['split_command', 'classify_source', 'compiler_language']
# Ignored compiler options map for compilation database creation.
-# The map is used in `split_command` method. (Which does ignore and classify
+# The map is used in `_split_command` method. (Which does ignore and classify
# parameters.) Please note, that these are not the only parameters which
# might be ignored.
#
@@ -48,61 +52,209 @@
'-Xlinker': 1
}
-# Known C/C++ compiler executable name patterns
-COMPILER_PATTERNS = frozenset([
- re.compile(r'^(intercept-|analyze-|)c(c|\+\+)$'),
- re.compile(r'^([^-]*-)*[mg](cc|\+\+)(-\d+(\.\d+){0,2})?$'),
- re.compile(r'^([^-]*-)*clang(\+\+)?(-\d+(\.\d+){0,2})?$'),
- re.compile(r'^llvm-g(cc|\+\+)$'),
+# Known C/C++ compiler wrapper name patterns
+COMPILER_PATTERN_WRAPPER = re.compile(r'^(distcc|ccache)$')
+
+# Known C compiler executable name patterns
+COMPILER_PATTERNS_CC = frozenset([
+ re.compile(r'^(|i|mpi)cc$'),
+ re.compile(r'^([^-]*-)*[mg]cc(-\d+(\.\d+){0,2})?$'),
+ re.compile(r'^([^-]*-)*clang(-\d+(\.\d+){0,2})?$'),
+ re.compile(r'^(g|)xlc$'),
])
+# Known C++ compiler executable name patterns
+COMPILER_PATTERNS_CXX = frozenset([
+ re.compile(r'^(c\+\+|cxx|CC)$'),
+ re.compile(r'^([^-]*-)*[mg]\+\+(-\d+(\.\d+){0,2})?$'),
+ re.compile(r'^([^-]*-)*clang\+\+(-\d+(\.\d+){0,2})?$'),
+ re.compile(r'^(icpc|mpiCC|mpicxx|mpic\+\+)$'),
+ re.compile(r'^(g|)xl(C|c\+\+)$'),
+])
-def split_command(command):
- """ Returns a value when the command is a compilation, None otherwise.
+CompilationCommand = collections.namedtuple(
+ 'CompilationCommand', ['compiler', 'flags', 'files'])
- The value on success is a named tuple with the following attributes:
- files: list of source files
- flags: list of compile options
- compiler: string value of 'c' or 'c++' """
+class Compilation:
+ def __init__(self, compiler, flags, source, directory):
+ """ Constructor for a single compilation.
- # the result of this method
- result = collections.namedtuple('Compilation',
- ['compiler', 'flags', 'files'])
- result.compiler = compiler_language(command)
- result.flags = []
- result.files = []
- # quit right now, if the program was not a C/C++ compiler
- if not result.compiler:
+ This method just normalize the paths and initialize values. """
+
+ self.compiler = compiler
+ self.flags = flags
+ self.directory = os.path.normpath(directory)
+ self.source = source if os.path.isabs(source) else \
+ os.path.normpath(os.path.join(self.directory, source))
+
+ def __hash__(self):
+ return hash((self.compiler, self.source, self.directory,
+ ':'.join(self.flags)))
+
+ def __eq__(self, other):
+ return isinstance(other, Compilation) and \
+ self.compiler == other.compiler and \
+ self.flags == other.flags and \
+ self.directory == other.directory and \
+ self.source == other.source
+
+ def as_dict(self):
+ """ This method dumps the object attributes into a dictionary. """
+
+ return dict((key, value) for key, value in vars(self).items())
+
+ def as_db_entry(self):
+ """ This method creates a compilation database entry. """
+
+ relative = os.path.relpath(self.source, self.directory)
+ compiler = 'cc' if self.compiler == 'c' else 'c++'
+ return {
+ 'file': relative,
+ 'arguments': [compiler, '-c'] + self.flags + [relative],
+ 'directory': self.directory
+ }
+
+ @staticmethod
+ def from_db_entry(entry):
+ """ Parser method for compilation entry.
+
+ From compilation database entry it creates the compilation object.
+
+ :param entry: the compilation database entry
+ :return: a single compilation object """
+
+ command = decode(entry['command']) if 'command' in entry else \
+ entry['arguments']
+ execution = Execution(cmd=command, cwd=entry['directory'], pid=0)
+ entries = list(Compilation.iter_from_execution(execution))
+ assert len(entries) == 1
+ return entries[0]
+
+ @staticmethod
+ def iter_from_execution(execution, cc='cc', cxx='c++'):
+ """ Generator method for compilation entries.
+
+ From a single compiler call it can generate zero or more entries.
+
+ :param execution: executed command and working directory
+ :param cc: user specified C compiler name
+ :param cxx: user specified C++ compiler name
+ :return: stream of CompilationDbEntry objects """
+
+ candidate = Compilation._split_command(execution.cmd, cc, cxx)
+ for source in (candidate.files if candidate else []):
+ result = Compilation(directory=execution.cwd,
+ source=source,
+ compiler=candidate.compiler,
+ flags=candidate.flags)
+ if os.path.isfile(result.source):
+ yield result
+
+ @staticmethod
+ def _split_compiler(command, cc, cxx):
+ """ A predicate to decide the command is a compiler call or not.
+
+ :param command: the command to classify
+ :param cc: user specified C compiler name
+ :param cxx: user specified C++ compiler name
+ :return: None if the command is not a compilation, or a tuple
+ (compiler_language, rest of the command) otherwise """
+
+ def is_wrapper(cmd):
+ return True if COMPILER_PATTERN_WRAPPER.match(cmd) else False
+
+ def is_c_compiler(cmd):
+ return os.path.basename(cc) == cmd or \
+ any(pattern.match(cmd) for pattern in COMPILER_PATTERNS_CC)
+
+ def is_cxx_compiler(cmd):
+ return os.path.basename(cxx) == cmd or \
+ any(pattern.match(cmd) for pattern in COMPILER_PATTERNS_CXX)
+
+ if command: # not empty list will allow to index '0' and '1:'
+ executable = os.path.basename(command[0])
+ parameters = command[1:]
+ # 'wrapper' 'parameters' and
+ # 'wrapper' 'compiler' 'parameters' are valid.
+ # plus, a wrapper can wrap wrapper too.
+ if is_wrapper(executable):
+ result = Compilation._split_compiler(parameters, cc, cxx)
+ return ('c', parameters) if result is None else result
+ # and 'compiler' 'parameters' is valid.
+ elif is_c_compiler(executable):
+ return 'c', parameters
+ elif is_cxx_compiler(executable):
+ return 'c++', parameters
return None
- # iterate on the compile options
- args = iter(command[1:])
- for arg in args:
- # quit when compilation pass is not involved
- if arg in {'-E', '-S', '-cc1', '-M', '-MM', '-###'}:
+
+ @staticmethod
+ def _split_command(command, cc, cxx):
+ """ Returns a value when the command is a compilation, None otherwise.
+
+ :param command: the command to classify
+ :param cc: user specified C compiler name
+ :param cxx: user specified C++ compiler name
+ :return: stream of CompilationCommand objects """
+
+ logging.debug('input was: %s', command)
+ # quit right now, if the program was not a C/C++ compiler
+ compiler_and_arguments = Compilation._split_compiler(command, cc, cxx)
+ if compiler_and_arguments is None:
return None
- # ignore some flags
- elif arg in IGNORED_FLAGS:
- count = IGNORED_FLAGS[arg]
- for _ in range(count):
- next(args)
- elif re.match(r'^-(l|L|Wl,).+', arg):
- pass
- # some parameters could look like filename, take as compile option
- elif arg in {'-D', '-I'}:
- result.flags.extend([arg, next(args)])
- # parameter which looks source file is taken...
- elif re.match(r'^[^-].+', arg) and classify_source(arg):
- result.files.append(arg)
- # and consider everything else as compile option.
- else:
- result.flags.append(arg)
- # do extra check on number of source files
- return result if result.files else None
+
+ # the result of this method
+ result = CompilationCommand(compiler=compiler_and_arguments[0],
+ flags=[],
+ files=[])
+ # iterate on the compile options
+ args = iter(compiler_and_arguments[1])
+ for arg in args:
+ # quit when compilation pass is not involved
+ if arg in {'-E', '-S', '-cc1', '-M', '-MM', '-###'}:
+ return None
+ # ignore some flags
+ elif arg in IGNORED_FLAGS:
+ count = IGNORED_FLAGS[arg]
+ for _ in range(count):
+ next(args)
+ elif re.match(r'^-(l|L|Wl,).+', arg):
+ pass
+ # some parameters could look like filename, take as compile option
+ elif arg in {'-D', '-I'}:
+ result.flags.extend([arg, next(args)])
+ # parameter which looks source file is taken...
+ elif re.match(r'^[^-].+', arg) and classify_source(arg):
+ result.files.append(arg)
+ # and consider everything else as compile option.
+ else:
+ result.flags.append(arg)
+ logging.debug('output is: %s', result)
+ # do extra check on number of source files
+ return result if result.files else None
+
+
+class CompilationDatabase:
+ @staticmethod
+ def save(filename, iterator):
+ entries = [entry.as_db_entry() for entry in iterator]
+ with open(filename, 'w+') as handle:
+ json.dump(entries, handle, sort_keys=True, indent=4)
+
+ @staticmethod
+ def load(filename):
+ with open(filename, 'r') as handle:
+ for entry in json.load(handle):
+ yield Compilation.from_db_entry(entry)
def classify_source(filename, c_compiler=True):
- """ Return the language from file name extension. """
+ """ Classify source file names and returns the presumed language,
+ based on the file name extension.
+
+ :param filename: the source file name
+ :param c_compiler: indicate that the compiler is a C compiler,
+ :return: the language from file name extension. """
mapping = {
'.c': 'c' if c_compiler else 'c++',
@@ -127,15 +279,28 @@
return mapping.get(extension)
+# Bellow this line, only temporary declarations for backward compatibility. #
+
+def split_command(command):
+ """ Returns a value when the command is a compilation, None otherwise.
+
+ The value on success is a named tuple with the following attributes:
+
+ files: list of source files
+ flags: list of compile options
+ compiler: string value of 'c' or 'c++' """
+
+ # the result of this method
+ return Compilation._split_command(command, "cc", "c++")
+
+
def compiler_language(command):
""" A predicate to decide the command is a compiler call or not.
Returns 'c' or 'c++' when it match. None otherwise. """
- cplusplus = re.compile(r'^(.+)(\+\+)(-.+|)$')
+ language_and_arguments = Compilation._split_compiler(command, "cc", "c++")
+ if language_and_arguments is None:
+ return None
- if command:
- executable = os.path.basename(command[0])
- if any(pattern.match(executable) for pattern in COMPILER_PATTERNS):
- return 'c++' if cplusplus.match(executable) else 'c'
- return None
+ return language_and_arguments[0]
Index: tools/scan-build-py/tests/unit/test_compilation.py
===================================================================
--- tools/scan-build-py/tests/unit/test_compilation.py
+++ tools/scan-build-py/tests/unit/test_compilation.py
@@ -10,88 +10,156 @@
class CompilerTest(unittest.TestCase):
- def test_is_compiler_call(self):
- self.assertIsNotNone(sut.compiler_language(['clang']))
- self.assertIsNotNone(sut.compiler_language(['clang-3.6']))
- self.assertIsNotNone(sut.compiler_language(['clang++']))
- self.assertIsNotNone(sut.compiler_language(['clang++-3.5.1']))
- self.assertIsNotNone(sut.compiler_language(['cc']))
- self.assertIsNotNone(sut.compiler_language(['c++']))
- self.assertIsNotNone(sut.compiler_language(['gcc']))
- self.assertIsNotNone(sut.compiler_language(['g++']))
- self.assertIsNotNone(sut.compiler_language(['/usr/local/bin/gcc']))
- self.assertIsNotNone(sut.compiler_language(['/usr/local/bin/g++']))
- self.assertIsNotNone(sut.compiler_language(['/usr/local/bin/clang']))
- self.assertIsNotNone(
- sut.compiler_language(['armv7_neno-linux-gnueabi-g++']))
-
- self.assertIsNone(sut.compiler_language([]))
- self.assertIsNone(sut.compiler_language(['']))
- self.assertIsNone(sut.compiler_language(['ld']))
- self.assertIsNone(sut.compiler_language(['as']))
- self.assertIsNone(sut.compiler_language(['/usr/local/bin/compiler']))
+ def assert_c_compiler(self, command, cc='nope', cxx='nope++'):
+ value = sut.Compilation._split_compiler(command, cc, cxx)
+ self.assertIsNotNone(value)
+ self.assertEqual(value[0], 'c')
+
+ def assert_cxx_compiler(self, command, cc='nope', cxx='nope++'):
+ value = sut.Compilation._split_compiler(command, cc, cxx)
+ self.assertIsNotNone(value)
+ self.assertEqual(value[0], 'c++')
+
+ def assert_not_compiler(self, command):
+ value = sut.Compilation._split_compiler(command, 'nope', 'nope')
+ self.assertIsNone(value)
+
+ def test_compiler_call(self):
+ self.assert_c_compiler(['cc'])
+ self.assert_cxx_compiler(['CC'])
+ self.assert_cxx_compiler(['c++'])
+ self.assert_cxx_compiler(['cxx'])
+
+ def test_clang_compiler_call(self):
+ self.assert_c_compiler(['clang'])
+ self.assert_c_compiler(['clang-3.6'])
+ self.assert_cxx_compiler(['clang++'])
+ self.assert_cxx_compiler(['clang++-3.5.1'])
+
+ def test_gcc_compiler_call(self):
+ self.assert_c_compiler(['gcc'])
+ self.assert_cxx_compiler(['g++'])
+
+ def test_intel_compiler_call(self):
+ self.assert_c_compiler(['icc'])
+ self.assert_cxx_compiler(['icpc'])
+
+ def test_aix_compiler_call(self):
+ self.assert_c_compiler(['xlc'])
+ self.assert_cxx_compiler(['xlc++'])
+ self.assert_cxx_compiler(['xlC'])
+ self.assert_c_compiler(['gxlc'])
+ self.assert_cxx_compiler(['gxlc++'])
+
+ def test_open_mpi_compiler_call(self):
+ self.assert_c_compiler(['mpicc'])
+ self.assert_cxx_compiler(['mpiCC'])
+ self.assert_cxx_compiler(['mpicxx'])
+ self.assert_cxx_compiler(['mpic++'])
+
+ def test_compiler_call_with_path(self):
+ self.assert_c_compiler(['/usr/local/bin/gcc'])
+ self.assert_cxx_compiler(['/usr/local/bin/g++'])
+ self.assert_c_compiler(['/usr/local/bin/clang'])
+
+ def test_cross_compiler_call(self):
+ self.assert_cxx_compiler(['armv7_neno-linux-gnueabi-g++'])
+
+ def test_compiler_wrapper_call(self):
+ self.assert_c_compiler(['distcc'])
+ self.assert_c_compiler(['distcc', 'cc'])
+ self.assert_cxx_compiler(['distcc', 'c++'])
+ self.assert_c_compiler(['ccache'])
+ self.assert_c_compiler(['ccache', 'cc'])
+ self.assert_cxx_compiler(['ccache', 'c++'])
+
+ def test_non_compiler_call(self):
+ self.assert_not_compiler([])
+ self.assert_not_compiler([''])
+ self.assert_not_compiler(['ld'])
+ self.assert_not_compiler(['as'])
+ self.assert_not_compiler(['/usr/local/bin/compiler'])
+
+ def test_specific_compiler_call(self):
+ self.assert_c_compiler(['nope'], cc='nope')
+ self.assert_c_compiler(['./nope'], cc='nope')
+ self.assert_c_compiler(['/path/nope'], cc='nope')
+ self.assert_cxx_compiler(['nope++'], cxx='nope++')
+ self.assert_cxx_compiler(['./nope++'], cxx='nope++')
+ self.assert_cxx_compiler(['/path/nope++'], cxx='nope++')
+
+ def assert_arguments_equal(self, expected, command):
+ value = sut.Compilation._split_compiler(command, 'nope', 'nope')
+ self.assertIsNotNone(value)
+ self.assertEqual(expected, value[1])
+
+ def test_argument_split(self):
+ arguments = ['-c', 'file.c']
+ self.assert_arguments_equal(arguments, ['distcc'] + arguments)
+ self.assert_arguments_equal(arguments, ['distcc', 'cc'] + arguments)
+ self.assert_arguments_equal(arguments, ['distcc', 'c++'] + arguments)
+ self.assert_arguments_equal(arguments, ['ccache'] + arguments)
+ self.assert_arguments_equal(arguments, ['ccache', 'cc'] + arguments)
+ self.assert_arguments_equal(arguments, ['ccache', 'c++'] + arguments)
class SplitTest(unittest.TestCase):
- def test_detect_cxx_from_compiler_name(self):
- def test(cmd):
- result = sut.split_command([cmd, '-c', 'src.c'])
- self.assertIsNotNone(result, "wrong input for test")
- return result.compiler == 'c++'
+ def assert_compilation(self, command):
+ result = sut.Compilation._split_command(command, 'nope', 'nope')
+ self.assertIsNotNone(result)
- self.assertFalse(test('cc'))
- self.assertFalse(test('gcc'))
- self.assertFalse(test('clang'))
-
- self.assertTrue(test('c++'))
- self.assertTrue(test('g++'))
- self.assertTrue(test('g++-5.3.1'))
- self.assertTrue(test('clang++'))
- self.assertTrue(test('clang++-3.7.1'))
- self.assertTrue(test('armv7_neno-linux-gnueabi-g++'))
+ def assert_non_compilation(self, command):
+ result = sut.Compilation._split_command(command, 'nope', 'nope')
+ self.assertIsNone(result)
def test_action(self):
- self.assertIsNotNone(sut.split_command(['clang', 'source.c']))
- self.assertIsNotNone(sut.split_command(['clang', '-c', 'source.c']))
- self.assertIsNotNone(sut.split_command(['clang', '-c', 'source.c',
- '-MF', 'a.d']))
+ self.assert_compilation(['clang', 'source.c'])
+ self.assert_compilation(['clang', '-c', 'source.c'])
+ self.assert_compilation(['clang', '-c', 'source.c', '-MF', 'a.d'])
+
+ self.assert_non_compilation(['clang', '-E', 'source.c'])
+ self.assert_non_compilation(['clang', '-c', '-E', 'source.c'])
+ self.assert_non_compilation(['clang', '-c', '-M', 'source.c'])
+ self.assert_non_compilation(['clang', '-c', '-MM', 'source.c'])
- self.assertIsNone(sut.split_command(['clang', '-E', 'source.c']))
- self.assertIsNone(sut.split_command(['clang', '-c', '-E', 'source.c']))
- self.assertIsNone(sut.split_command(['clang', '-c', '-M', 'source.c']))
- self.assertIsNone(
- sut.split_command(['clang', '-c', '-MM', 'source.c']))
+ def assert_source_files(self, expected, command):
+ result = sut.Compilation._split_command(command, 'nope', 'nope')
+ self.assertIsNotNone(result)
+ self.assertEqual(expected, result.files)
def test_source_file(self):
- def test(expected, cmd):
- self.assertEqual(expected, sut.split_command(cmd).files)
-
- test(['src.c'], ['clang', 'src.c'])
- test(['src.c'], ['clang', '-c', 'src.c'])
- test(['src.C'], ['clang', '-x', 'c', 'src.C'])
- test(['src.cpp'], ['clang++', '-c', 'src.cpp'])
- test(['s1.c', 's2.c'], ['clang', '-c', 's1.c', 's2.c'])
- test(['s1.c', 's2.c'], ['cc', 's1.c', 's2.c', '-ldep', '-o', 'a.out'])
- test(['src.c'], ['clang', '-c', '-I', './include', 'src.c'])
- test(['src.c'], ['clang', '-c', '-I', '/opt/me/include', 'src.c'])
- test(['src.c'], ['clang', '-c', '-D', 'config=file.c', 'src.c'])
-
- self.assertIsNone(
- sut.split_command(['cc', 'this.o', 'that.o', '-o', 'a.out']))
- self.assertIsNone(
- sut.split_command(['cc', 'this.o', '-lthat', '-o', 'a.out']))
+ self.assert_source_files(['src.c'], ['clang', 'src.c'])
+ self.assert_source_files(['src.c'], ['clang', '-c', 'src.c'])
+ self.assert_source_files(['src.C'], ['clang', '-x', 'c', 'src.C'])
+ self.assert_source_files(['src.cpp'], ['clang++', '-c', 'src.cpp'])
+ self.assert_source_files(['s1.c', 's2.c'],
+ ['clang', '-c', 's1.c', 's2.c'])
+ self.assert_source_files(['s1.c', 's2.c'],
+ ['cc', 's1.c', 's2.c', '-ldp', '-o', 'a.out'])
+ self.assert_source_files(['src.c'],
+ ['clang', '-c', '-I', './include', 'src.c'])
+ self.assert_source_files(['src.c'],
+ ['clang', '-c', '-I', '/opt/inc', 'src.c'])
+ self.assert_source_files(['src.c'],
+ ['clang', '-c', '-Dconfig=file.c', 'src.c'])
+
+ self.assert_non_compilation(['cc', 'this.o', 'that.o', '-o', 'a.out'])
+ self.assert_non_compilation(['cc', 'this.o', '-lthat', '-o', 'a.out'])
+
+ def assert_flags(self, expected, flags):
+ command = ['clang', '-c', 'src.c'] + flags
+ result = sut.Compilation._split_command(command, 'nope', 'nope')
+ self.assertIsNotNone(result)
+ self.assertEqual(expected, result.flags)
def test_filter_flags(self):
- def test(expected, flags):
- command = ['clang', '-c', 'src.c'] + flags
- self.assertEqual(expected, sut.split_command(command).flags)
def same(expected):
- test(expected, expected)
+ self.assert_flags(expected, expected)
def filtered(flags):
- test([], flags)
+ self.assert_flags([], flags)
same([])
same(['-I', '/opt/me/include', '-DNDEBUG', '-ULIMITS'])
@@ -108,15 +176,36 @@
class SourceClassifierTest(unittest.TestCase):
+ def assert_non_source(self, filename):
+ result = sut.classify_source(filename)
+ self.assertIsNone(result)
+
+ def assert_c_source(self, filename, force):
+ result = sut.classify_source(filename, force)
+ self.assertEqual('c', result)
+
+ def assert_cxx_source(self, filename, force):
+ result = sut.classify_source(filename, force)
+ self.assertEqual('c++', result)
+
def test_sources(self):
- self.assertIsNone(sut.classify_source('file.o'))
- self.assertIsNone(sut.classify_source('file.exe'))
- self.assertIsNone(sut.classify_source('/path/file.o'))
- self.assertIsNone(sut.classify_source('clang'))
-
- self.assertEqual('c', sut.classify_source('file.c'))
- self.assertEqual('c', sut.classify_source('./file.c'))
- self.assertEqual('c', sut.classify_source('/path/file.c'))
- self.assertEqual('c++', sut.classify_source('file.c', False))
- self.assertEqual('c++', sut.classify_source('./file.c', False))
- self.assertEqual('c++', sut.classify_source('/path/file.c', False))
+ self.assert_non_source('file.o')
+ self.assert_non_source('file.exe')
+ self.assert_non_source('/path/file.o')
+ self.assert_non_source('clang')
+
+ self.assert_c_source('file.c', True)
+ self.assert_cxx_source('file.c', False)
+
+ self.assert_cxx_source('file.cxx', True)
+ self.assert_cxx_source('file.cxx', False)
+ self.assert_cxx_source('file.c++', True)
+ self.assert_cxx_source('file.c++', False)
+ self.assert_cxx_source('file.cpp', True)
+ self.assert_cxx_source('file.cpp', False)
+
+ self.assert_c_source('/path/file.c', True)
+ self.assert_c_source('./path/file.c', True)
+ self.assert_c_source('../path/file.c', True)
+ self.assert_c_source('/file.c', True)
+ self.assert_c_source('./file.c', True)