[scan-build-py] Add sarif-html support in scan-build-py

[isthismyaccount]

Summary

Update scan-build-py to be able to trigger sarif-html output format in clang static analyzer.

NOTE: testcase test_sarif_and_html_creates_sarif_and_html_reports will fail if the default clang does not have change https://reviews.llvm.org/D96389 . This can be remediated by pointing the default clang in arguments.py to a locally built clang. I was unable to figure out where these particular tests for scan-build-py are being invoked (aside from manually), so any help there would be greatly appreciated.

[aabbaabb]

lgtm

[isthismyaccount]

Hi Gabor, do you mind taking a look at this? Would love to get a couple more eyes on this change since we're not super familiar with the code.

[xazax.hun]

LGTM!

It would be nice, if you could also open a PR for the github version: https://github.com/rizsotto/scan-build

Unfortunately, we have two versions of scan-build-py that are slightly different, but it would be nice if the two would have feature parity :)

[isthismyaccount]

@xazax.hun Happy to do so, it occurred to me that this change is predicated by this: https://reviews.llvm.org/D94251 which I did not have the foresight to ask you to review. When you have the chance, do you mind taking a look at that since it's a little more substantial than this one? If both look good, I'll attempt to add both changes to the other scan-build.

clang/tools/scan-build-py/libscanbuild/analyze.py

def report_directory(hint, keep, output_format):
     name = tempfile.mkdtemp(prefix=stamp, dir=parent_dir)
 
     logging.info('Report directory created: %s', name)
 
     try:
         yield name
     finally:
         if os.listdir(name):
-            if output_format != 'sarif':
+            if output_format not in ['sarif', 'sarif-html']: # FIXME:
                 # 'scan-view' currently does not support sarif format.
                 msg = "Run 'scan-view %s' to examine bug reports."
+            elif output_format == 'sarif-html':
+                msg = "Run 'scan-view %s' to examine bug reports or see " \
+                    "merged sarif results at %s/results-merged.sarif."
             else:
-                msg = "View result at %s/results-merged.sarif."
+                msg = "View merged sarif results at %s/results-merged.sarif."
             keep = True
         else:
             if keep:
                 msg = "Report directory '%s' contains no report, but kept."
             else:
                 msg = "Removing directory '%s' because it contains no report."
         logging.warning(msg, name)
 

 
 @require(['command',  # entry from compilation database
           'directory',  # entry from compilation database
           'file',  # entry from compilation database
           'clang',  # clang executable name (and path)
           'direct_args',  # arguments from command line
           'force_debug',  # kill non debug macros
           'output_dir',  # where generated report files shall go
-          'output_format',  # it's 'plist', 'html', 'plist-html', 'plist-multi-file', or 'sarif'
+          'output_format',  # it's 'plist', 'html', 'plist-html', 'plist-multi-file', 'sarif', or 'sarif-html'
           'output_failures',  # generate crash reports or not
           'ctu'])  # ctu control options
 def run(opts):
     """ Entry point to run (or not) static analyzer against a single entry
     of the compilation database.
 
     This complex task is decomposed into smaller methods which are calling
     each other in chain. If the analysis is not possible the given method
def target():
                 'plist',
                 'plist-html',
                 'plist-multi-file'}:
             (handle, name) = tempfile.mkstemp(prefix='report-',
                                               suffix='.plist',
                                               dir=opts['output_dir'])
             os.close(handle)
             return name
-        elif opts['output_format'] == 'sarif':
+        elif opts['output_format'] in {
+                'sarif',
+                'sarif-html'}:
             (handle, name) = tempfile.mkstemp(prefix='result-',
                                               suffix='.sarif',
                                               dir=opts['output_dir'])
             os.close(handle)
             return name
         return opts['output_dir']
 
     try:

clang/tools/scan-build-py/libscanbuild/arguments.py

def create_analyze_parser(from_build_command):
     format_group.add_argument(
         '--sarif',
         '-sarif',
         dest='output_format',
         const='sarif',
         default='html',
         action='store_const',
         help="""Cause the results as a result.sarif file.""")
+    format_group.add_argument(
+        '--sarif-html',
+        '-sarif-html',
+        dest='output_format',
+        const='sarif-html',
+        default='html',
+        action='store_const',
+        help="""Cause the results as a result.sarif file and .html files.""")
 
     advanced = parser.add_argument_group('advanced options')
     advanced.add_argument(
         '--use-analyzer',
         metavar='<path>',
         dest='clang',
         default='clang',
         help="""'%(prog)s' uses the 'clang' executable relative to itself for

clang/tools/scan-build-py/libscanbuild/report.py

 from libscanbuild.clang import get_version
 
 __all__ = ['document']
 
 
 def document(args):
     """ Generates cover report and returns the number of bugs/crashes. """
 
-    html_reports_available = args.output_format in {'html', 'plist-html'}
-    sarif_reports_available = args.output_format in {'sarif'}
+    html_reports_available = args.output_format in {'html', 'plist-html', 'sarif-html'}
+    sarif_reports_available = args.output_format in {'sarif', 'sarif-html'}
 
     logging.debug('count crashes and bugs')
     crash_count = sum(1 for _ in read_crashes(args.output))
     bug_counter = create_counters()
     for bug in read_bugs(args.output, html_reports_available):
         bug_counter(bug)
     result = crash_count + bug_counter.total
 

clang/tools/scan-build-py/tests/functional/cases/test_from_cdb.py

class OutputFormatTest(unittest.TestCase):
     @staticmethod
     def get_html_count(directory):
         return len(glob.glob(os.path.join(directory, 'report-*.html')))
 
     @staticmethod
     def get_plist_count(directory):
         return len(glob.glob(os.path.join(directory, 'report-*.plist')))
 
-    def test_default_creates_html_report(self):
+    @staticmethod
+    def get_sarif_count(directory):
+        return len(glob.glob(os.path.join(directory, 'result-*.sarif')))
+
+    def test_default_only_creates_html_report(self):
         with libear.TemporaryDirectory() as tmpdir:
             cdb = prepare_cdb('regular', tmpdir)
             exit_code, reportdir = run_analyzer(tmpdir, cdb, [])
             self.assertTrue(
                 os.path.exists(os.path.join(reportdir, 'index.html')))
             self.assertEqual(self.get_html_count(reportdir), 2)
             self.assertEqual(self.get_plist_count(reportdir), 0)
+            self.assertEqual(self.get_sarif_count(reportdir), 0)
 
-    def test_plist_and_html_creates_html_report(self):
+    def test_plist_and_html_creates_html_and_plist_reports(self):
         with libear.TemporaryDirectory() as tmpdir:
             cdb = prepare_cdb('regular', tmpdir)
             exit_code, reportdir = run_analyzer(tmpdir, cdb, ['--plist-html'])
             self.assertTrue(
                 os.path.exists(os.path.join(reportdir, 'index.html')))
             self.assertEqual(self.get_html_count(reportdir), 2)
             self.assertEqual(self.get_plist_count(reportdir), 5)
+            self.assertEqual(self.get_sarif_count(reportdir), 0)
 
-    def test_plist_does_not_creates_html_report(self):
+    def test_plist_only_creates_plist_report(self):
         with libear.TemporaryDirectory() as tmpdir:
             cdb = prepare_cdb('regular', tmpdir)
             exit_code, reportdir = run_analyzer(tmpdir, cdb, ['--plist'])
             self.assertFalse(
                 os.path.exists(os.path.join(reportdir, 'index.html')))
             self.assertEqual(self.get_html_count(reportdir), 0)
             self.assertEqual(self.get_plist_count(reportdir), 5)
+            self.assertEqual(self.get_sarif_count(reportdir), 0)
+
+    def test_sarif_only_creates_sarif_result(self):
+        with libear.TemporaryDirectory() as tmpdir:
+            cdb = prepare_cdb('regular', tmpdir)
+            exit_code, reportdir = run_analyzer(tmpdir, cdb, ['--sarif'])
+            self.assertFalse(
+                os.path.exists(os.path.join(reportdir, 'index.html')))
+            self.assertTrue(
+                os.path.exists(os.path.join(reportdir, 'results-merged.sarif')))
+            self.assertEqual(self.get_html_count(reportdir), 0)
+            self.assertEqual(self.get_plist_count(reportdir), 0)
+            self.assertEqual(self.get_sarif_count(reportdir), 5)
+
+    def test_sarif_and_html_creates_sarif_and_html_reports(self):
+        with libear.TemporaryDirectory() as tmpdir:
+            cdb = prepare_cdb('regular', tmpdir)
+            exit_code, reportdir = run_analyzer(tmpdir, cdb, ['--sarif-html'])
+            self.assertTrue(
+                os.path.exists(os.path.join(reportdir, 'index.html')))
+            self.assertTrue(
+                os.path.exists(os.path.join(reportdir, 'results-merged.sarif')))
+            self.assertEqual(self.get_html_count(reportdir), 2)
+            self.assertEqual(self.get_plist_count(reportdir), 0)
+            self.assertEqual(self.get_sarif_count(reportdir), 5)
 
 
 class FailureReportTest(unittest.TestCase):
     def test_broken_creates_failure_reports(self):
         with libear.TemporaryDirectory() as tmpdir:
             cdb = prepare_cdb('broken', tmpdir)
             exit_code, reportdir = run_analyzer(tmpdir, cdb, [])
             self.assertTrue(