[scan-build-py] Prevent crashes of CTU analysis by suppressing warnings

[lorincbalog]

Summary

During the mapping of functions for cross-translation unit analysis, the functions' names and the containing files are collected by clang-func-mapping. Warnings are stored in the collection too, which later makes the parsing of the collection fail. Proposed solution is to suppress all warnings during the function mapping.

We are about to create a buildbot to test the CTU changes in Clang by analyzing some open-source projects. At the moment we are blocked by this issue in scan-build-py, because the warnings are causing analyze-build to fail even on Xerces-C++ .

[whisperity]

Will this properly synergise across compilers with user-specified warning options, such as -Wall -Werror?

[lorincbalog]

Yes, -w suppresses all warnings (without raising an error) regardless of the options' sequence, even if -Wall -Werror are present,.

[george.karpenkov]

If that helps you, then sure.

I'm not sure I understand why having warnings causes the collection process to fail, but I guess ultimately it's not important.

tools/scan-build-py/libscanbuild/analyze.py

def report_failure(opts):
 
     # Classify error type: when Clang terminated by a signal it's a 'Crash'.
     # (python subprocess Popen.returncode is negative when child terminated
     # by signal.) Everything else is 'Other Error'.
     error = 'crash' if opts['exit_code'] < 0 else 'other_error'
     # Create preprocessor output file name. (This is blindly following the
     # Perl implementation.)
     (handle, name) = tempfile.mkstemp(suffix=extension(),
-                                      prefix='clang_' + error + '_',
-                                      dir=destination())
-    os.close(handle)
-    # Execute Clang again, but run the syntax check only.
-    cwd = opts['directory']
-    cmd = get_arguments(
-        [opts['clang'], '-fsyntax-only', '-E'
-         ] + opts['flags'] + [opts['file'], '-o', name], cwd)
-    run_command(cmd, cwd=cwd)
-    # write general information about the crash
-    with open(name + '.info.txt', 'w') as handle:
-        handle.write(opts['file'] + os.linesep)
-        handle.write(error.title().replace('_', ' ') + os.linesep)
-        handle.write(' '.join(cmd) + os.linesep)
-        handle.write(' '.join(os.uname()) + os.linesep)
-        handle.write(get_version(opts['clang']))
-        handle.close()
-    # write the captured output too
-    with open(name + '.stderr.txt', 'w') as handle:
-        handle.writelines(opts['error_output'])
-        handle.close()
+                                      prefix='clang_' + error + '_',
+                                      dir=destination())
+    os.close(handle)
+    # Execute Clang again, but run the syntax check only.
+    cwd = opts['directory']
+    cmd = get_arguments(
+        [opts['clang'], '-fsyntax-only', '-E'
+         ] + opts['flags'] + [opts['file'], '-o', name], cwd)
+    run_command(cmd, cwd=cwd)
+    # write general information about the crash
+    with open(name + '.info.txt', 'w') as handle:
+        handle.write(opts['file'] + os.linesep)
+        handle.write(error.title().replace('_', ' ') + os.linesep)
+        handle.write(' '.join(cmd) + os.linesep)
+        handle.write(' '.join(os.uname()) + os.linesep)
+        handle.write(get_version(opts['clang']))
+        handle.close()
+    # write the captured output too
+    with open(name + '.stderr.txt', 'w') as handle:
+        handle.writelines(opts['error_output'])
+        handle.close()
+
 
-
-@require(['clang', 'directory', 'flags', 'direct_args', 'file', 'output_dir',
-          'output_format'])
-def run_analyzer(opts, continuation=report_failure):
-    """ It assembles the analysis command line and executes it. Capture the
-    output of the analysis and returns with it. If failure reports are
-    requested, it calls the continuation to generate it. """
-
-    def target():
-        """ Creates output file name for reports. """
-        if opts['output_format'] in {
-                'plist',
-                'plist-html',
-                'plist-multi-file'}:
-            (handle, name) = tempfile.mkstemp(prefix='report-',
-                                              suffix='.plist',
-                                              dir=opts['output_dir'])
-            os.close(handle)
-            return name
-        return opts['output_dir']
-
-    try:
-        cwd = opts['directory']
-        cmd = get_arguments([opts['clang'], '--analyze'] +
-                            opts['direct_args'] + opts['flags'] +
-                            [opts['file'], '-o', target()],
-                            cwd)
-        output = run_command(cmd, cwd=cwd)
-        return {'error_output': output, 'exit_code': 0}
-    except subprocess.CalledProcessError as ex:
-        result = {'error_output': ex.output, 'exit_code': ex.returncode}
-        if opts.get('output_failures', False):
-            opts.update(result)
-            continuation(opts)
-        return result
+@require(['clang', 'directory', 'flags', 'direct_args', 'file', 'output_dir',
+          'output_format'])
+def run_analyzer(opts, continuation=report_failure):
+    """ It assembles the analysis command line and executes it. Capture the
+    output of the analysis and returns with it. If failure reports are
+    requested, it calls the continuation to generate it. """
+
+    def target():
+        """ Creates output file name for reports. """
+        if opts['output_format'] in {
+                'plist',
+                'plist-html',
+                'plist-multi-file'}:
+            (handle, name) = tempfile.mkstemp(prefix='report-',
+                                              suffix='.plist',
+                                              dir=opts['output_dir'])
+            os.close(handle)
+            return name
+        return opts['output_dir']
+
+    try:
+        cwd = opts['directory']
+        cmd = get_arguments([opts['clang'], '--analyze'] +
+                            opts['direct_args'] + opts['flags'] +
+                            [opts['file'], '-o', target()],
+                            cwd)
+        output = run_command(cmd, cwd=cwd)
+        return {'error_output': output, 'exit_code': 0}
+    except subprocess.CalledProcessError as ex:
+        result = {'error_output': ex.output, 'exit_code': ex.returncode}
+        if opts.get('output_failures', False):
+            opts.update(result)
+            continuation(opts)
+        return result
+
 
-
-def func_map_list_src_to_ast(func_src_list):
-    """ Turns textual function map list with source files into a
-    function map list with ast files. """
-
-    func_ast_list = []
-    for fn_src_txt in func_src_list:
-        mangled_name, path = fn_src_txt.split(" ", 1)
-        # Normalize path on windows as well
-        path = os.path.splitdrive(path)[1]
-        # Make relative path out of absolute
-        path = path[1:] if path[0] == os.sep else path
-        ast_path = os.path.join("ast", path + ".ast")
-        func_ast_list.append(mangled_name + " " + ast_path)
-    return func_ast_list
+def func_map_list_src_to_ast(func_src_list):
+    """ Turns textual function map list with source files into a
+    function map list with ast files. """
+
+    func_ast_list = []
+    for fn_src_txt in func_src_list:
+        mangled_name, path = fn_src_txt.split(" ", 1)
+        # Normalize path on windows as well
+        path = os.path.splitdrive(path)[1]
+        # Make relative path out of absolute
+        path = path[1:] if path[0] == os.sep else path
+        ast_path = os.path.join("ast", path + ".ast")
+        func_ast_list.append(mangled_name + " " + ast_path)
+    return func_ast_list
+
 
-
-@require(['clang', 'directory', 'flags', 'direct_args', 'file', 'ctu'])
-def ctu_collect_phase(opts):
-    """ Preprocess source by generating all data needed by CTU analysis. """
-
-    def generate_ast(triple_arch):
-        """ Generates ASTs for the current compilation command. """
-
-        args = opts['direct_args'] + opts['flags']
-        ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
-                                       os.path.realpath(opts['file'])[1:] +
-                                       '.ast')
-        ast_path = os.path.abspath(ast_joined_path)
-        ast_dir = os.path.dirname(ast_path)
-        if not os.path.isdir(ast_dir):
-            try:
-                os.makedirs(ast_dir)
-            except OSError:
-                # In case an other process already created it.
-                pass
-        ast_command = [opts['clang'], '-emit-ast']
-        ast_command.extend(args)
-        ast_command.append('-w')
-        ast_command.append(opts['file'])
-        ast_command.append('-o')
-        ast_command.append(ast_path)
-        logging.debug("Generating AST using '%s'", ast_command)
-        run_command(ast_command, cwd=opts['directory'])
-
-    def map_functions(triple_arch):
-        """ Generate function map file for the current source. """
-
-        args = opts['direct_args'] + opts['flags']
-        funcmap_command = [opts['ctu'].func_map_cmd]
-        funcmap_command.append(opts['file'])
-        funcmap_command.append('--')
-        funcmap_command.extend(args)
+@require(['clang', 'directory', 'flags', 'direct_args', 'file', 'ctu'])
+def ctu_collect_phase(opts):
+    """ Preprocess source by generating all data needed by CTU analysis. """
+
+    def generate_ast(triple_arch):
+        """ Generates ASTs for the current compilation command. """
+
+        args = opts['direct_args'] + opts['flags']
+        ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
+                                       os.path.realpath(opts['file'])[1:] +
+                                       '.ast')
+        ast_path = os.path.abspath(ast_joined_path)
+        ast_dir = os.path.dirname(ast_path)
+        if not os.path.isdir(ast_dir):
+            try:
+                os.makedirs(ast_dir)
+            except OSError:
+                # In case an other process already created it.
+                pass
+        ast_command = [opts['clang'], '-emit-ast']
+        ast_command.extend(args)
+        ast_command.append('-w')
+        ast_command.append(opts['file'])
+        ast_command.append('-o')
+        ast_command.append(ast_path)
+        logging.debug("Generating AST using '%s'", ast_command)
+        run_command(ast_command, cwd=opts['directory'])
+
+    def map_functions(triple_arch):
+        """ Generate function map file for the current source. """
+
+        args = opts['direct_args'] + opts['flags']
+        funcmap_command = [opts['ctu'].func_map_cmd]
+        funcmap_command.append(opts['file'])
+        funcmap_command.append('--')
+        funcmap_command.extend(args)
+        funcmap_command.append('-w')
         logging.debug("Generating function map using '%s'", funcmap_command)
         func_src_list = run_command(funcmap_command, cwd=opts['directory'])
         func_ast_list = func_map_list_src_to_ast(func_src_list)
         extern_fns_map_folder = os.path.join(opts['ctu'].dir, triple_arch,
                                              CTU_TEMP_FNMAP_FOLDER)
         if not os.path.isdir(extern_fns_map_folder):
             try:
                 os.makedirs(extern_fns_map_folder)
if ctu_config.collect or ctu_config.analyze:
         if ctu_config.analyze:
             cwd = opts['directory']
             cmd = [opts['clang'], '--analyze'] + opts['direct_args'] \
                 + opts['flags'] + [opts['file']]
             triarch = get_triple_arch(cmd, cwd)
             ctu_options = ['ctu-dir=' + os.path.join(ctu_config.dir, triarch),
                            'experimental-enable-naive-ctu-analysis=true']
             analyzer_options = prefix_with('-analyzer-config', ctu_options)
             direct_options = prefix_with('-Xanalyzer', analyzer_options)
 
     return continuation(opts)
 
 
 @require(['flags', 'force_debug'])
 def filter_debug_flags(opts, continuation=dispatch_ctu):
     """ Filter out nondebug macros when requested. """