diff --git a/clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h b/clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h --- a/clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h +++ b/clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h @@ -141,6 +141,9 @@ llvm::Optional ConditionalVariant; }; +/// Drops comment tokens. +TokenStream stripComments(const TokenStream &); + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp b/clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp --- a/clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp +++ b/clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp @@ -201,6 +201,17 @@ OSTREAM_DUMP(PPStructure::Code) #undef OSTREAM_DUMP +TokenStream stripComments(const TokenStream &Input) { + TokenStream Out; + for (const Token &T : Input.tokens()) { + if (T.Kind == tok::comment) + continue; + Out.push(T); + } + Out.finalize(); + return Out; +} + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp b/clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp --- a/clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp +++ b/clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp @@ -23,11 +23,16 @@ using testing::_; using testing::ElementsAre; +using testing::ElementsAreArray; using testing::Matcher; using testing::Pair; using testing::StrEq; using Chunk = PPStructure::Chunk; +MATCHER_P2(token, Text, Kind, "") { + return arg.Kind == Kind && arg.text() == Text; +} + MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) { std::vector Texts; for (const Token &Tok : TS.tokens(arg.Tokens)) @@ -146,6 +151,22 @@ EXPECT_EQ(0u, X.End.Tokens.size()); } +TEST(Preprocessor, StripComments) { + LangOptions Opts; + std::string Code = R"cpp( + // comment + int; +)cpp"; + TokenStream Raw = cook(lex(Code, Opts), Opts); + TokenStream Stripped = stripComments(Raw); + EXPECT_THAT(Raw.tokens(), ElementsAreArray({token("// comment", tok::comment), + token("int", tok::kw_int), + token(";", tok::semi)})); + + EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int), + token(";", tok::semi)})); +} + } // namespace } // namespace pseudo } // namespace syntax