diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -127,6 +127,7 @@ clangToolingCore clangToolingInclusions clangToolingRefactoring + clangToolingSyntax ${LLVM_PTHREAD_LIB} ${CLANGD_ATOMIC_LIB} ) diff --git a/clang-tools-extra/clangd/ClangdUnit.h b/clang-tools-extra/clangd/ClangdUnit.h --- a/clang-tools-extra/clangd/ClangdUnit.h +++ b/clang-tools-extra/clangd/ClangdUnit.h @@ -24,6 +24,7 @@ #include "clang/Serialization/ASTBitCodes.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" +#include "clang/Tooling/Syntax/Tokens.h" #include #include #include @@ -115,10 +116,14 @@ const IncludeStructure &getIncludeStructure() const; const CanonicalIncludes &getCanonicalIncludes() const; + /// Tokens recorded while parsing the main file. + /// (!) does not have tokens from the preamble. + const syntax::TokenBuffer &tokens() const { return Tokens; } + private: ParsedAST(std::shared_ptr Preamble, std::unique_ptr Clang, - std::unique_ptr Action, + std::unique_ptr Action, syntax::TokenBuffer Tokens, std::vector LocalTopLevelDecls, std::vector Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes); @@ -132,6 +137,9 @@ // FrontendAction.EndSourceFile). std::unique_ptr Clang; std::unique_ptr Action; + /// Expanded tokens for the main file. Does not contain tokens for the file + /// preamble. + syntax::TokenBuffer Tokens; // Data, stored after parsing. std::vector Diags; diff --git a/clang-tools-extra/clangd/ClangdUnit.cpp b/clang-tools-extra/clangd/ClangdUnit.cpp --- a/clang-tools-extra/clangd/ClangdUnit.cpp +++ b/clang-tools-extra/clangd/ClangdUnit.cpp @@ -36,6 +36,7 @@ #include "clang/Serialization/ASTWriter.h" #include "clang/Serialization/PCHContainerOperations.h" #include "clang/Tooling/CompilationDatabase.h" +#include "clang/Tooling/Syntax/Tokens.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" @@ -418,6 +419,9 @@ collectIWYUHeaderMaps(&CanonIncludes); Clang->getPreprocessor().addCommentHandler(IWYUHandler.get()); + // Collect tokens of the main file. + syntax::TokenCollector Tokens(Clang->getPreprocessor()); + if (!Action->Execute()) log("Execute() failed when building AST for {0}", MainInput.getFile()); @@ -446,8 +450,9 @@ if (Preamble) Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end()); return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action), - std::move(ParsedDecls), std::move(Diags), - std::move(Includes), std::move(CanonIncludes)); + std::move(Tokens).consume(), std::move(ParsedDecls), + std::move(Diags), std::move(Includes), + std::move(CanonIncludes)); } ParsedAST::ParsedAST(ParsedAST &&Other) = default; @@ -540,11 +545,13 @@ ParsedAST::ParsedAST(std::shared_ptr Preamble, std::unique_ptr Clang, std::unique_ptr Action, + syntax::TokenBuffer Tokens, std::vector LocalTopLevelDecls, std::vector Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes) : Preamble(std::move(Preamble)), Clang(std::move(Clang)), - Action(std::move(Action)), Diags(std::move(Diags)), + Action(std::move(Action)), Tokens(std::move(Tokens)), + Diags(std::move(Diags)), LocalTopLevelDecls(std::move(LocalTopLevelDecls)), Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) { assert(this->Clang); diff --git a/clang-tools-extra/clangd/tool/CMakeLists.txt b/clang-tools-extra/clangd/tool/CMakeLists.txt --- a/clang-tools-extra/clangd/tool/CMakeLists.txt +++ b/clang-tools-extra/clangd/tool/CMakeLists.txt @@ -26,5 +26,6 @@ clangSema clangTooling clangToolingCore + clangToolingSyntax ${CLANGD_XPC_LIBS} ) diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -86,6 +86,7 @@ clangTooling clangToolingCore clangToolingInclusions + clangToolingSyntax LLVMSupport LLVMTestingSupport )