diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -61,6 +61,7 @@ clang-refactor clang-diff clang-scan-deps + clang-syntax diagtool hmaptool ) diff --git a/clang/test/clang-syntax/Inputs/foodir b/clang/test/clang-syntax/Inputs/foodir new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/foodir @@ -0,0 +1 @@ +// A C++ header with same name as that of a directory in the include path. diff --git a/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/Headers/Framework.h b/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/Headers/Framework.h new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/Headers/Framework.h @@ -0,0 +1,2 @@ +// This comment is stripped, so size is changed when file is opened +#define FRAMEWORK 0 diff --git a/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h b/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h @@ -0,0 +1,2 @@ +// This comment is stripped when file is opened, so size will change +#define PRIV 0 diff --git a/clang/test/clang-syntax/Inputs/has_include_if_elif.json b/clang/test/clang-syntax/Inputs/has_include_if_elif.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/has_include_if_elif.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/has_include_if_elif2.cpp -IInputs", + "file": "DIR/has_include_if_elif2.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/header.h b/clang/test/clang-syntax/Inputs/header.h new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/header.h @@ -0,0 +1,3 @@ +#ifdef INCLUDE_HEADER2 +#include "header2.h" +#endif diff --git a/clang/test/clang-syntax/Inputs/header2.h b/clang/test/clang-syntax/Inputs/header2.h new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/header2.h @@ -0,0 +1 @@ +// header 2. diff --git a/clang/test/clang-syntax/Inputs/header_stat_before_open_cdb.json b/clang/test/clang-syntax/Inputs/header_stat_before_open_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/header_stat_before_open_cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/header_stat_before_open_input.m -iframework Inputs/frameworks", + "file": "DIR/header_stat_before_open_input.m" +} +] diff --git a/clang/test/clang-syntax/Inputs/headerwithdirname.json b/clang/test/clang-syntax/Inputs/headerwithdirname.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/headerwithdirname.json @@ -0,0 +1,7 @@ +[ + { + "directory": "DIR", + "command": "clang -c -IDIR -IDIR/foodir -IInputs DIR/headerwithdirname_input.cpp", + "file": "DIR/headerwithdirname_input.cpp" + } +] diff --git a/clang/test/clang-syntax/Inputs/headerwithdirnamefollowedbyinclude.json b/clang/test/clang-syntax/Inputs/headerwithdirnamefollowedbyinclude.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/headerwithdirnamefollowedbyinclude.json @@ -0,0 +1,7 @@ +[ + { + "directory": "DIR", + "command": "clang -c -IDIR -IInputs DIR/headerwithdirname_input.cpp", + "file": "DIR/headerwithdirname_input.cpp" + } +] diff --git a/clang/test/clang-syntax/Inputs/module.modulemap b/clang/test/clang-syntax/Inputs/module.modulemap new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/module.modulemap @@ -0,0 +1,7 @@ +module header1 { + header "header.h" +} + +module header2 { + header "header2.h" +} diff --git a/clang/test/clang-syntax/Inputs/modules_cdb.json b/clang/test/clang-syntax/Inputs/modules_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/modules_cdb.json @@ -0,0 +1,22 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "file": "DIR/modules_cdb_input2.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o", + "file": "DIR/modules_cdb_input.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/no-werror.json b/clang/test/clang-syntax/Inputs/no-werror.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/no-werror.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/no-werror_input.cpp -IInputs -std=c++17 -Weverything -Werror", + "file": "DIR/no-werror.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/regular_cdb.json b/clang/test/clang-syntax/Inputs/regular_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/regular_cdb.json @@ -0,0 +1,17 @@ +[ +{ + "directory": "DIR", + "command": "clang -E -fsyntax-only DIR/regular_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/regular_cdb2.d", + "file": "DIR/regular_cdb_input2.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/regular_cdb_input.cpp -IInputs", + "file": "DIR/regular_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/regular_cdb_input.cpp -IInputs -o adena.o", + "file": "DIR/regular_cdb_input.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/static-analyzer-cdb.json b/clang/test/clang-syntax/Inputs/static-analyzer-cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/static-analyzer-cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang --analyze DIR/static-analyzer.c", + "file": "DIR/static-analyzer.c" +} +] diff --git a/clang/test/clang-syntax/Inputs/strip_diag_serialize.json b/clang/test/clang-syntax/Inputs/strip_diag_serialize.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/strip_diag_serialize.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E -fsyntax-only DIR/strip_diag_serialize_input.cpp --serialize-diagnostics /does/not/exist", + "file": "DIR/strip_diag_serialize_input.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/subframework_header_dir_symlink_cdb.json b/clang/test/clang-syntax/Inputs/subframework_header_dir_symlink_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/subframework_header_dir_symlink_cdb.json @@ -0,0 +1,12 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/subframework_header_dir_symlink_input.m -D EMPTY -iframework Inputs/frameworks", + "file": "DIR/subframework_header_dir_symlink_input.m" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/subframework_header_dir_symlink_input2.m -FInputs/frameworks -iframework Inputs/frameworks_symlink/../frameworks_symlink", + "file": "DIR/subframework_header_dir_symlink_input2.m" +} +] diff --git a/clang/test/clang-syntax/Inputs/symlink_cdb.json b/clang/test/clang-syntax/Inputs/symlink_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/symlink_cdb.json @@ -0,0 +1,12 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/symlink_input.cpp -IInputs", + "file": "DIR/symlink_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/symlink_input2.cpp -IInputs", + "file": "DIR/symlink_input2.cpp" +} +] diff --git a/clang/test/clang-syntax/Inputs/sys-header.h b/clang/test/clang-syntax/Inputs/sys-header.h new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/sys-header.h @@ -0,0 +1 @@ +#define MACRO 201411 diff --git a/clang/test/clang-syntax/Inputs/vfsoverlay.yaml b/clang/test/clang-syntax/Inputs/vfsoverlay.yaml new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/vfsoverlay.yaml @@ -0,0 +1,12 @@ +{ + 'version': 0, + 'roots': [ + { 'name': 'DIR', 'type': 'directory', + 'contents': [ + { 'name': 'not_real.h', 'type': 'file', + 'external-contents': 'DIR/Inputs/header.h' + } + ] + } + ] +} diff --git a/clang/test/clang-syntax/Inputs/vfsoverlay_cdb.json b/clang/test/clang-syntax/Inputs/vfsoverlay_cdb.json new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/Inputs/vfsoverlay_cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/vfsoverlay_input.cpp -IInputs -ivfsoverlay DIR/vfsoverlay.yaml", + "file": "DIR/vfsoverlay_input.cpp" +} +] diff --git a/clang/test/clang-syntax/no_args.cpp b/clang/test/clang-syntax/no_args.cpp new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/no_args.cpp @@ -0,0 +1,15 @@ +// RUN: rm -rf %t.dir +// RUN: mkdir -p %t.dir +// +// RUN: clang-syntax 2> %t.dir/no_arg_errs || true +// RUN: echo EOF >> %t.dir/no_arg_errs +// +// RUN: FileCheck %s --input-file %t.dir/no_arg_errs + +int main() { + return 0; +} + +// CHECK: Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found. +// CHECK-EMPTY: +// CHECK-NEXT: EOF diff --git a/clang/test/clang-syntax/syntax_hello_world.cpp b/clang/test/clang-syntax/syntax_hello_world.cpp new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/syntax_hello_world.cpp @@ -0,0 +1,29 @@ +// RUN: rm -rf %t.dir +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/hello_world.cpp +// +// RUN: clang-syntax -dump-syntax %t.dir/hello_world.cpp > %t.dir/syntax_hello_world +// RUN: echo EOF >> %t.dir/syntax_hello_world +// RUN: FileCheck %s --input-file %t.dir/syntax_hello_world + +int main() { + return 0; +} + +// CHECK: *: TranslationUnit +// CHECK-NEXT: `-SimpleDeclaration +// CHECK-NEXT: |-int +// CHECK-NEXT: |-SimpleDeclarator +// CHECK-NEXT: | |-main +// CHECK-NEXT: | `-ParametersAndQualifiers +// CHECK-NEXT: | |-( +// CHECK-NEXT: | `-) +// CHECK-NEXT: `-CompoundStatement +// CHECK-NEXT: |-{ +// CHECK-NEXT: |-ReturnStatement +// CHECK-NEXT: | |-return +// CHECK-NEXT: | |-UnknownExpression +// CHECK-NEXT: | | `-0 +// CHECK-NEXT: | `-; +// CHECK-NEXT: `-} +// CHECK-NEXT: EOF diff --git a/clang/test/clang-syntax/syntax_no_file_arg.cpp b/clang/test/clang-syntax/syntax_no_file_arg.cpp new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/syntax_no_file_arg.cpp @@ -0,0 +1,10 @@ +// RUN: rm -rf %t.dir +// RUN: mkdir -p %t.dir +// +// RUN: clang-syntax -dump-syntax 2> %t.dir/syntax_no_file_arg_errs || true +// RUN: echo EOF >> %t.dir/syntax_no_file_arg_errs +// RUN: FileCheck %s --input-file %t.dir/syntax_no_file_arg_errs + +// CHECK: Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found. +// CHECK-EMPTY: +// CHECK-NEXT: EOF diff --git a/clang/test/clang-syntax/tokens_hello_world.cpp b/clang/test/clang-syntax/tokens_hello_world.cpp new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/tokens_hello_world.cpp @@ -0,0 +1,19 @@ +// RUN: rm -rf %t.dir +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/hello_world.cpp +// +// RUN: clang-syntax -dump-tokens %t.dir/hello_world.cpp > %t.dir/tokens_hello_world +// RUN: echo EOF >> %t.dir/tokens_hello_world +// RUN: FileCheck %s --input-file %t.dir/tokens_hello_world + +int main() { + return 0; +} + +// CHECK: expanded tokens: +// CHECK-NEXT: int main ( ) { return 0 ; } +// CHECK-NEXT: tools/clang/test/clang-syntax/Output/tokens_hello_world.cpp.tmp.dir/hello_world.cpp +// CHECK-NEXT: spelled tokens: +// CHECK-NEXT: int main ( ) { return 0 ; } +// CHECK-NEXT: no mappings. +// CHECK-NEXT: EOF diff --git a/clang/test/clang-syntax/tokens_no_file_arg.cpp b/clang/test/clang-syntax/tokens_no_file_arg.cpp new file mode 100644 --- /dev/null +++ b/clang/test/clang-syntax/tokens_no_file_arg.cpp @@ -0,0 +1,10 @@ +// RUN: rm -rf %t.dir +// RUN: mkdir -p %t.dir +// +// RUN: clang-syntax -dump-tokens 2> %t.dir/tokens_no_file_arg_errs || true +// RUN: echo EOF >> %t.dir/tokens_no_file_arg_errs +// RUN: FileCheck %s --input-file %t.dir/tokens_no_file_arg_errs + +// CHECK: Failed to create 'standalone': [StandaloneToolExecutorPlugin] No positional argument found. +// CHECK-EMPTY: +// CHECK-NEXT: EOF diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt --- a/clang/tools/CMakeLists.txt +++ b/clang/tools/CMakeLists.txt @@ -18,6 +18,7 @@ if(UNIX) add_clang_subdirectory(clang-shlib) endif() +add_clang_subdirectory(clang-syntax) if(CLANG_ENABLE_ARCMT) add_clang_subdirectory(arcmt-test) diff --git a/clang/tools/clang-syntax/CMakeLists.txt b/clang/tools/clang-syntax/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang/tools/clang-syntax/CMakeLists.txt @@ -0,0 +1,16 @@ +set(LLVM_LINK_COMPONENTS Support) + +add_clang_tool(clang-syntax + SyntaxMain.cpp + ) + +target_link_libraries(clang-syntax + PRIVATE + clangAST + clangBasic + clangFrontend + clangLex + clangTooling + clangToolingCore + clangToolingSyntax +) diff --git a/clang/tools/clang-syntax/SyntaxMain.cpp b/clang/tools/clang-syntax/SyntaxMain.cpp new file mode 100644 --- /dev/null +++ b/clang/tools/clang-syntax/SyntaxMain.cpp @@ -0,0 +1,88 @@ +//===- SyntaxMain.cpp -----------------------------------------*- C++ -*-=====// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Tooling/Execution.h" +#include "clang/Tooling/Syntax/BuildTree.h" +#include "clang/Tooling/Syntax/Tokens.h" +#include "clang/Tooling/Syntax/Tree.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" +#include + +using namespace clang; + +namespace { + +llvm::cl::OptionCategory ClangSyntaxOptions("clang-syntax common options"); + +llvm::cl::opt DumpTokens("dump-tokens", + llvm::cl::desc("dump the preprocessed tokens"), + llvm::cl::init(false), + llvm::cl::cat(ClangSyntaxOptions)); +llvm::cl::opt DumpSyntax("dump-syntax", + llvm::cl::desc("dump the syntax tree"), + llvm::cl::init(false), + llvm::cl::cat(ClangSyntaxOptions)); + +class BuildSyntaxTree : public ASTFrontendAction { +public: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + class Consumer : public ASTConsumer { + public: + Consumer(CompilerInstance &CI) : Collector(CI.getPreprocessor()) {} + + void HandleTranslationUnit(ASTContext &AST) override { + syntax::Arena A(AST.getSourceManager(), AST.getLangOpts(), + std::move(Collector).consume()); + auto *TU = syntax::buildSyntaxTree(A, *AST.getTranslationUnitDecl()); + if (DumpTokens) + llvm::outs() << A.tokenBuffer().dumpForTests(); + if (DumpSyntax) + llvm::outs() << TU->dump(A); + } + + private: + syntax::TokenCollector Collector; + }; + return std::make_unique(CI); + } +}; + +class Factory : public tooling::FrontendActionFactory { + std::unique_ptr create() override { + return std::make_unique(); + } +}; + +} // namespace + +int main(int argc, const char **argv) { + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + auto Executor = clang::tooling::createExecutorFromCommandLineArgs( + argc, argv, ClangSyntaxOptions, + "Build syntax trees for the specified files"); + if (!Executor) { + llvm::errs() << llvm::toString(Executor.takeError()) << "\n"; + return 1; + } + + if (!DumpTokens && !DumpSyntax) { + llvm::errs() + << "Please specify at least one of -dump-tree or -dump-tokens\n"; + return 1; + } + // Collect symbols found in each translation unit, merging as we go. + auto Err = Executor->get()->execute(std::make_unique()); + if (Err) + llvm::errs() << llvm::toString(std::move(Err)) << "\n"; + return 0; +}