[clang-tidy] added JSON output to clang-tidy-diff.py

[kuhnel]

Summary

This makes it easier to process the clang-tidy output in other tools.
For far it's only parsing the findings, not the output on stderr.

For testing you can use:

bash
ninja clangd
COMMIT=4dedd82cc99341d757a9cc07a8b7b22c8bb61d19
git diff $COMMIT~ $COMMIT | python3 clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py -p1 -json -checks="*"

This is part of the effort towards re-enabling clang-tidy in
pre-merge testing. With JSON output, we can deprecate some local
code in pre-merge testing.

https://github.com/google/llvm-premerge-checks/issues/367

[shafik]

Is this change still relevant of can we close this PR?

clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py

 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import threading
 import traceback
+from typing import Optional
 
 try:
   import yaml
 except ImportError:
   yaml = None
 
 is_py2 = sys.version[0] == '2'
 
 if is_py2:
     import Queue as queue
 else:
     import queue as queue
 
 
-def run_tidy(task_queue, lock, timeout):
+def run_tidy(task_queue, lock, timeout, result_queue: Optional[queue.Queue]):
   watchdog = None
   while True:
     command = task_queue.get()
     try:
       proc = subprocess.Popen(command,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
 
       if timeout is not None:
         watchdog = threading.Timer(timeout, proc.kill)
         watchdog.start()
 
       stdout, stderr = proc.communicate()
 
       with lock:
+        if result_queue:
+          parse_clang_tidy_output(result_queue, stdout.decode('utf-8'))
+        else:
           sys.stdout.write(stdout.decode('utf-8') + '\n')
           sys.stdout.flush()
         if stderr:
           sys.stderr.write(stderr.decode('utf-8') + '\n')
           sys.stderr.flush()
     except Exception as e:
       with lock:
         sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
     finally:
       with lock:
         if not (timeout is None or watchdog is None):
           if not watchdog.is_alive():
               sys.stderr.write('Terminated by timeout: ' +
                                ' '.join(command) + '\n')
           watchdog.cancel()
       task_queue.task_done()
 
 
-def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
+def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout, result_queue):
   for _ in range(max_tasks):
-    t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
+    t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout, result_queue))
     t.daemon = True
     t.start()
 
 
 def merge_replacement_files(tmpdir, mergefile):
   """Merge all replacement files in a directory into a single file"""
   # The fixes suggested by clang-tidy >= 4.0.0 are given under
   # the top level key 'Diagnostics' in the output yaml files
if merged:
     output = {'MainSourceFile': '', mergekey: merged}
     with open(mergefile, 'w') as out:
       yaml.safe_dump(output, out)
   else:
     # Empty the file:
     open(mergefile, 'w').close()
 
 
+def parse_clang_tidy_output(result_queue: queue, clang_tidy_output: str):
+  """Parse the clang-tidy output and add them to the result queue."""
+  _regex_pattern = r"^(?P<path>/[^:]+):\W*(?P<line>\d+):\W*(?P<character>\d+):\W*(?P<check>.*)$"
+  for finding in re.finditer(_regex_pattern, clang_tidy_output, re.MULTILINE):
+    result_queue.put({
+        'path': finding['path'],
+        'line': int(finding['line']),
+        'character': int(finding['character']),
+        'check': finding['check'],
+    })
+
+
 def main():
   parser = argparse.ArgumentParser(description=
                                    'Run clang-tidy against changed files, and '
                                    'output diagnostics only for modified '
                                    'lines.')
   parser.add_argument('-clang-tidy-binary', metavar='PATH',
                       default='clang-tidy',
                       help='path to clang-tidy binary')
parser.add_argument('-extra-arg', dest='extra_arg',
                       help='Additional argument to append to the compiler '
                       'command line.')
   parser.add_argument('-extra-arg-before', dest='extra_arg_before',
                       action='append', default=[],
                       help='Additional argument to prepend to the compiler '
                       'command line.')
   parser.add_argument('-quiet', action='store_true', default=False,
                       help='Run clang-tidy in quiet mode')
+  parser.add_argument('-json', action='store_true', dest='json_output',
+                      help='Output clang-tidy results in JSON format.')
   clang_tidy_args = []
   argv = sys.argv[1:]
   if '--' in argv:
     clang_tidy_args.extend(argv[argv.index('--'):])
     argv = argv[:argv.index('--')]
 
   args = parser.parse_args(argv)
 
def main():
   if yaml and args.export_fixes:
     tmpdir = tempfile.mkdtemp()
 
   # Tasks for clang-tidy.
   task_queue = queue.Queue(max_task_count)
   # A lock for console output.
   lock = threading.Lock()
 
+  # if we want JSON output: gather results
+  result_queue = None
+  if args.json_output:
+    result_queue = queue.Queue()
+
   # Run a pool of clang-tidy workers.
-  start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
+  start_workers(max_task_count, run_tidy, task_queue,
+                lock, args.timeout, result_queue)
 
   # Form the common args list.
   common_clang_tidy_args = []
   if args.fix:
     common_clang_tidy_args.append('-fix')
   if args.checks != '':
     common_clang_tidy_args.append('-checks=' + args.checks)
   if args.quiet:
     common_clang_tidy_args.append('-quiet')
   if args.build_path is not None:
     common_clang_tidy_args.append('-p=%s' % args.build_path)
-  if args.use_color:
+  if args.use_color and not args.json_output:
     common_clang_tidy_args.append('--use-color')
   for arg in args.extra_arg:
     common_clang_tidy_args.append('-extra-arg=%s' % arg)
   for arg in args.extra_arg_before:
     common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
 
   for name in lines_by_file:
     line_filter_json = json.dumps(
try:
       merge_replacement_files(tmpdir, args.export_fixes)
     except:
       sys.stderr.write('Error exporting fixes.\n')
       traceback.print_exc()
 
   if tmpdir:
     shutil.rmtree(tmpdir)
 
-
+  if args.json_output:
+    json_output = {'findings': list(result_queue.queue)} 
+    json.dump(json_output, sys.stdout, indent=2)
 if __name__ == '__main__':
   main()