[clang-format] Add run-clang-format.py script.

[curdeius]

Summary

Fixes https://github.com/llvm/llvm-project/issues/62108.

NOTE: Clang-Format Team Automated Review Comment

It looks like your clang-format review does not contain any unit tests, please try to ensure all code changes have a unit test (unless this is an NFC or refactoring, adding documentation etc..)

Add your unit tests in clang/unittests/Format and you can build with ninja FormatTests. We recommend using the verifyFormat(xxx) format of unit tests rather than EXPECT_EQ as this will ensure you change is tolerant to random whitespace changes (see FormatTest.cpp as an example)

For situations where your change is altering the TokenAnnotator.cpp which can happen if you are trying to improve the annotation phase to ensure we are correctly identifying the type of a token, please add a token annotator test in TokenAnnotatorTest.cpp

[curdeius]

I guess I need to add a test in a similar that run-clang-tidy is tested, but I'd like to gather some initial review still.

[Eugene.Zelenko]

clang/tools/run-clang-format.py

1

Looks like Windows en of lines were mixed with UNIX ones.

26

Is it really necessary since Python 3 is specified explicitly?

30

Should be before os.

[MyDeveloperDay]

is it possible to pass other arguments like -n if not running in place is there any protection from prevent stdout/stderr from overlapping..?

[MyDeveloperDay]

clang/tools/run-clang-format.py

63

what about

run-clang-format `<run-clang-format-arg> --  <clang-format-args>

clang/tools/CMakeLists.txt

 # It also may be included by LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR.
 add_llvm_external_project(clang-tools-extra extra)
 
 # libclang may require clang-tidy in clang-tools-extra.
 add_clang_subdirectory(libclang)
 
 add_clang_subdirectory(amdgpu-arch)
 add_clang_subdirectory(nvptx-arch)
+
+install(PROGRAMS run-clang-format.py
+  DESTINATION "${CMAKE_INSTALL_BINDIR}"
+  COMPONENT clang-format
+  RENAME run-clang-format)

clang/tools/run-clang-format.py

+#!/usr/bin/env python3
+#
+#===- run-clang-format.py - Parallel clang-format runner ----*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===-----------------------------------------------------------------------===#
+
+"""
+Parallel clang-format runner
+==========================
+
+Runs clang-format over all files in given directories. Requires clang-format in $PATH.
+
+Example invocations.
+- Run clang-format on all files in the current working directory.
+    run-clang-format.py
+
+- Run clang-format on all files in the chosen directories.
+    run-clang-format.py dir1 dir2 dir3
+"""
+
+
+from __future__ import print_function
+import argparse
+import fnmatch
+import os
+import multiprocessing
+import queue
+import subprocess
+import sys
+import threading
+
+
+def glob_files(args):
+    files = []
+
+    extensions = args.extensions.split(',')
+
+    for directory in args.directories:
+        for root, _, filenames in os.walk(directory):
+            for ext in extensions:
+                for filename in fnmatch.filter(filenames, '*.' + ext):
+                    files.append(os.path.join(root, filename))
+
+    return files
+
+
+def parse_args(argv=None):
+    if argv is None:
+        argv = sys.argv
+    parser = argparse.ArgumentParser(
+        description='Runs clang-format over all files in given directories.'
+        ' Requires clang-format in PATH.')
+    parser.add_argument('--clang-format-binary', metavar='PATH',
+                        default='clang-format',
+                        help='path to clang-format binary')
+    parser.add_argument('-e', '--extensions', dest='extensions',
+                        help='comma-delimited list of extensions used to glob source files',
+                        default="c,cc,cpp,cxx,c++,h,hh,hpp,hxx,h++")
+    parser.add_argument('-style',
+                        help='formatting style',
+                        default="file")
+    parser.add_argument('--no-inplace', dest='inplace', action='store_false',
+                        help='do not format files inplace, but write output to the console'
+                        ' (useful for debugging)',
+                        default=True)
+    parser.add_argument('-j', metavar='THREAD_COUNT', type=int, default=0,
+                        help='number of clang-format instances to be run in parallel')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='output verbose comments')
+    parser.add_argument(metavar='DIRPATH', dest='directories', nargs='*',
+                        help='path(s) used to glob source files')
+
+    args = parser.parse_args(argv[1:])
+
+    if not args.directories:
+        args.directories = [os.getcwd()]
+
+    check_clang_format_binary(args)
+
+    return args
+
+
+def _get_format_invocation(args, filename):
+    invocation = [args.clang_format_binary]
+    invocation.append('-style=' + args.style)
+    if args.inplace:
+        invocation.append('-i')
+
+    invocation.append(filename)
+    return invocation
+
+
+def check_clang_format_binary(args):
+    """Checks if invoking supplied clang-format binary works."""
+    try:
+        subprocess.check_output([args.clang_format_binary, '--version'])
+    except OSError:
+        print('Unable to run clang-format. Is clang-format '
+              'binary correctly specified?', file=sys.stderr)
+        raise
+
+
+def run_format(args, task_queue, formatted_files):
+    """Takes filenames out of queue and runs clang-format on them."""
+    while True:
+        filename = task_queue.get()
+        invocation = _get_format_invocation(args, filename)
+
+        if args.verbose:
+            print('Processing {}'.format(filename))
+        formatted = subprocess.check_output(invocation)
+        formatted_files[filename] = formatted
+
+        task_queue.task_done()
+
+
+def format_all(args, files):
+    max_task = args.j
+    if max_task == 0:
+        max_task = multiprocessing.cpu_count()
+
+    formatted_files = {}
+
+    try:
+        # Spin up a bunch of format-launching threads.
+        task_queue = queue.Queue(max_task)
+        for _ in range(max_task):
+            task_thread = threading.Thread(target=run_format,
+                                           args=(args, task_queue, formatted_files))
+            task_thread.daemon = True
+            task_thread.start()
+
+        # Fill the queue with files.
+        for name in files:
+            task_queue.put(name)
+
+        # Wait for all threads to be done.
+        task_queue.join()
+
+    except OSError:
+        print("Cannot find clang-format at '{}'.".format(args.clang_format_binary),
+              file=sys.stderr)
+        raise
+
+    except subprocess.CalledProcessError as ex:
+        print("Running clang-format failed with non-zero status.", file=sys.stderr)
+        print("Command    : {}".format(' '.join(ex.cmd)), file=sys.stderr)
+        print("Return code: {}".format(str(ex.returncode)), file=sys.stderr)
+        raise
+
+    return formatted_files
+
+
+def main():
+    args = parse_args()
+
+    files = glob_files(args)
+
+    format_all(args, files)
+
+
+if __name__ == '__main__':
+    main()  # pragma: no cover