diff --git a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h --- a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h +++ b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h @@ -195,6 +195,9 @@ /// (And having cooked token kinds in PP-disabled sections is useful for us). TokenStream cook(const TokenStream &, const clang::LangOptions &); +/// Drops comment tokens. +TokenStream stripComments(const TokenStream &); + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp --- a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp +++ b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp @@ -93,6 +93,17 @@ OS << '\n'; } +TokenStream stripComments(const TokenStream &Input) { + TokenStream Out; + for (const Token &T : Input.tokens()) { + if (T.Kind == tok::comment) + continue; + Out.push(T); + } + Out.finalize(); + return Out; +} + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp --- a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp +++ b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp @@ -172,6 +172,23 @@ })); } +TEST(TokenTest, DropComments) { + LangOptions Opts; + std::string Code = R"cpp( + // comment + int /*abc*/; +)cpp"; + TokenStream Raw = cook(lex(Code, Opts), Opts); + TokenStream Stripped = stripComments(Raw); + EXPECT_THAT(Raw.tokens(), + ElementsAreArray( + {token("// comment", tok::comment), token("int", tok::kw_int), + token("/*abc*/", tok::comment), token(";", tok::semi)})); + + EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int), + token(";", tok::semi)})); +} + } // namespace } // namespace pseudo } // namespace syntax