[Syntax] A tool to dump syntax tree and token buffer

[hlopko]

Summary

Taking over of https://reviews.llvm.org/D70788.

[hlopko]

Cleanup

[hlopko]

Cleanup

[hlopko]

Talked to gribozavr2 offline and decided to abandon this patch. If in the future we'll have a need for a tool to dump syntax trees, we'll probably implement it in clang behind hidden options, and we'll provide a json output format as well.

clang/test/CMakeLists.txt

list(APPEND CLANG_TEST_DEPS
   clang-format
   clang-tblgen
   clang-offload-bundler
   clang-import-test
   clang-rename
   clang-refactor
   clang-diff
   clang-scan-deps
+  clang-syntax
   diagtool
   hmaptool
   )
   
 if(CLANG_ENABLE_STATIC_ANALYZER)
   list(APPEND CLANG_TEST_DEPS
     clang-check
     clang-extdef-mapping

clang/test/clang-syntax/no_args.cpp

+// RUN: rm -rf %t.dir
+// RUN: mkdir -p %t.dir
+//
+// RUN: clang-syntax 2> %t.dir/no_arg_errs || true
+// RUN: echo EOF >> %t.dir/no_arg_errs
+//
+// RUN: FileCheck %s --input-file %t.dir/no_arg_errs
+
+int main() {
+  return 0;
+}
+
+// CHECK:      Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found.
+// CHECK-EMPTY:
+// CHECK-NEXT: EOF

clang/test/clang-syntax/syntax_hello_world.cpp

+// RUN: rm -rf %t.dir
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/hello_world.cpp
+//
+// RUN: clang-syntax -dump-syntax %t.dir/hello_world.cpp > %t.dir/syntax_hello_world
+// RUN: echo EOF >> %t.dir/syntax_hello_world
+// RUN: FileCheck %s --input-file %t.dir/syntax_hello_world
+
+int main() {
+  return 0;
+}
+
+// CHECK:      *: TranslationUnit
+// CHECK-NEXT: `-SimpleDeclaration
+// CHECK-NEXT:   |-int
+// CHECK-NEXT:   |-SimpleDeclarator
+// CHECK-NEXT:   | |-main
+// CHECK-NEXT:   | `-ParametersAndQualifiers
+// CHECK-NEXT:   |   |-(
+// CHECK-NEXT:   |   `-)
+// CHECK-NEXT:   `-CompoundStatement
+// CHECK-NEXT:     |-{
+// CHECK-NEXT:     |-ReturnStatement
+// CHECK-NEXT:     | |-return
+// CHECK-NEXT:     | |-UnknownExpression
+// CHECK-NEXT:     | | `-0
+// CHECK-NEXT:     | `-;
+// CHECK-NEXT:     `-}
+// CHECK-NEXT: EOF

clang/test/clang-syntax/syntax_no_file_arg.cpp

+// RUN: rm -rf %t.dir
+// RUN: mkdir -p %t.dir
+//
+// RUN: clang-syntax -dump-syntax 2> %t.dir/syntax_no_file_arg_errs || true
+// RUN: echo EOF >> %t.dir/syntax_no_file_arg_errs
+// RUN: FileCheck %s --input-file %t.dir/syntax_no_file_arg_errs
+
+// CHECK:      Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found.
+// CHECK-EMPTY:
+// CHECK-NEXT: EOF

clang/test/clang-syntax/tokens_hello_world.cpp

+// RUN: rm -rf %t.dir
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/hello_world.cpp
+//
+// RUN: clang-syntax -dump-tokens %t.dir/hello_world.cpp > %t.dir/tokens_hello_world
+// RUN: echo EOF >> %t.dir/tokens_hello_world
+// RUN: FileCheck %s --input-file %t.dir/tokens_hello_world
+
+int main() {
+  return 0;
+}
+
+// CHECK:      expanded tokens:
+// CHECK-NEXT:   int main ( ) { return 0 ; }
+// CHECK-NEXT: tools/clang/test/clang-syntax/Output/tokens_hello_world.cpp.tmp.dir/hello_world.cpp
+// CHECK-NEXT:   spelled tokens:
+// CHECK-NEXT:     int main ( ) { return 0 ; }
+// CHECK-NEXT:   no mappings.
+// CHECK-NEXT: EOF

clang/test/clang-syntax/tokens_no_file_arg.cpp

+// RUN: rm -rf %t.dir
+// RUN: mkdir -p %t.dir
+//
+// RUN: clang-syntax -dump-tokens 2> %t.dir/tokens_no_file_arg_errs || true
+// RUN: echo EOF >> %t.dir/tokens_no_file_arg_errs
+// RUN: FileCheck %s --input-file %t.dir/tokens_no_file_arg_errs
+
+// CHECK:      Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found.
+// CHECK-EMPTY:
+// CHECK-NEXT: EOF

clang/tools/CMakeLists.txt

 
 add_clang_subdirectory(c-index-test)
 
 add_clang_subdirectory(clang-rename)
 add_clang_subdirectory(clang-refactor)
 if(UNIX)
   add_clang_subdirectory(clang-shlib)
 endif()
+add_clang_subdirectory(clang-syntax)
 
 if(CLANG_ENABLE_ARCMT)
   add_clang_subdirectory(arcmt-test)
   add_clang_subdirectory(c-arcmt-test)
 endif()
 
 if(CLANG_ENABLE_STATIC_ANALYZER)
   add_clang_subdirectory(clang-check)

clang/tools/clang-syntax/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS Support)
+
+add_clang_tool(clang-syntax
+  SyntaxMain.cpp
+  )
+
+target_link_libraries(clang-syntax
+  PRIVATE
+  clangAST
+  clangBasic
+  clangFrontend
+  clangLex
+  clangTooling
+  clangToolingCore
+  clangToolingSyntax
+)

clang/tools/clang-syntax/SyntaxMain.cpp

+//===- SyntaxMain.cpp -----------------------------------------*- C++ -*-=====//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/Execution.h"
+#include "clang/Tooling/Syntax/BuildTree.h"
+#include "clang/Tooling/Syntax/Tokens.h"
+#include "clang/Tooling/Syntax/Tree.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+using namespace clang;
+
+namespace {
+
+llvm::cl::OptionCategory ClangSyntaxOptions("clang-syntax common options");
+
+llvm::cl::opt<bool> DumpTokens("dump-tokens",
+                               llvm::cl::desc("dump the preprocessed tokens"),
+                               llvm::cl::init(false),
+                               llvm::cl::cat(ClangSyntaxOptions));
+llvm::cl::opt<bool> DumpSyntax("dump-syntax",
+                               llvm::cl::desc("dump the syntax tree"),
+                               llvm::cl::init(false),
+                               llvm::cl::cat(ClangSyntaxOptions));
+
+class BuildSyntaxTree : public ASTFrontendAction {
+public:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override {
+    class Consumer : public ASTConsumer {
+    public:
+      Consumer(CompilerInstance &CI) : Collector(CI.getPreprocessor()) {}
+
+      void HandleTranslationUnit(ASTContext &AST) override {
+        syntax::Arena A(AST.getSourceManager(), AST.getLangOpts(),
+                        std::move(Collector).consume());
+        auto *TU = syntax::buildSyntaxTree(A, *AST.getTranslationUnitDecl());
+        if (DumpTokens)
+          llvm::outs() << A.tokenBuffer().dumpForTests();
+        if (DumpSyntax)
+          llvm::outs() << TU->dump(A);
+      }
+
+    private:
+      syntax::TokenCollector Collector;
+    };
+    return std::make_unique<Consumer>(CI);
+  }
+};
+
+class Factory : public tooling::FrontendActionFactory {
+  std::unique_ptr<FrontendAction> create() override {
+    return std::make_unique<BuildSyntaxTree>();
+  }
+};
+
+} // namespace
+
+int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  auto Executor = clang::tooling::createExecutorFromCommandLineArgs(
+      argc, argv, ClangSyntaxOptions,
+      "Build syntax trees for the specified files");
+  if (!Executor) {
+    llvm::errs() << llvm::toString(Executor.takeError()) << "\n";
+    return 1;
+  }
+
+  if (!DumpTokens && !DumpSyntax) {
+    llvm::errs()
+        << "Please specify at least one of -dump-tree or -dump-tokens\n";
+    return 1;
+  }
+  // Collect symbols found in each translation unit, merging as we go.
+  auto Err = Executor->get()->execute(std::make_unique<Factory>());
+  if (Err)
+    llvm::errs() << llvm::toString(std::move(Err)) << "\n";
+  return 0;
+}