[pseudo] Strip comments for TokenStream.

[hokein]

Summary

Add a utility function to strip comments from a "raw" tokenstream. The
derived stream will be fed to the GLR parser (for early testing).

[sammccall]

clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h

144

I think this fits more naturally into Token.h next to cook().

It produces and consumes a token stream, and its implementation is a trivial use of those APIs.

While formally we might see comment stripping as part of the preprocessor, this file is more to do with directive structure. (Maybe it should be renamed?)

clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp

158

also include a /*comment*/ between int and ;?

[hokein]

clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h

144

moving to Token.h sounds good to me -- I'd treat this is a trivial implementation (I just wrote it to test the GLR parser). And yeah, stripping comment is just part of preprocessing the code.

Maybe it should be renamed?

The current Preprocess.h file name indicates that everything about preprocessing of TokenStream should live in it.

Since this file defines PPStructure, just call it PPStructure.h/cpp for better fine grained granularity?

[hokein]

address comment.

clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h

 
 private:
   // Wasteful, a union variant would be better!
   llvm::Optional<Code> CodeVariant;
   llvm::Optional<Directive> DirectiveVariant;
   llvm::Optional<Conditional> ConditionalVariant;
 };
 
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang
 
 #endif

clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp

#define OSTREAM_DUMP(Type)                                                     \
   }
 OSTREAM_DUMP(PPStructure)
 OSTREAM_DUMP(PPStructure::Chunk)
 OSTREAM_DUMP(PPStructure::Directive)
 OSTREAM_DUMP(PPStructure::Conditional)
 OSTREAM_DUMP(PPStructure::Code)
 #undef OSTREAM_DUMP
 
+TokenStream stripComments(const TokenStream &Input) {
+  TokenStream Out;
+  for (const Token &T : Input.tokens()) {
+    if (T.Kind == tok::comment)
+      continue;
+    Out.push(T);
+  }
+  Out.finalize();
+  return Out;
+}
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang

clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp

 
 namespace clang {
 namespace syntax {
 namespace pseudo {
 namespace {
 
 using testing::_;
 using testing::ElementsAre;
+using testing::ElementsAreArray;
 using testing::Matcher;
 using testing::Pair;
 using testing::StrEq;
 using Chunk = PPStructure::Chunk;
 
+MATCHER_P2(token, Text, Kind, "") {
+  return arg.Kind == Kind && arg.text() == Text;
+}
+
 MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
   std::vector<llvm::StringRef> Texts;
   for (const Token &Tok : TS.tokens(arg.Tokens))
     Texts.push_back(Tok.text());
   return Matcher<std::string>(StrEq(Tokens))
       .MatchAndExplain(llvm::join(Texts, " "), result_listener);
 }
 
)cpp";
   EXPECT_EQ(tok::pp_if, X.Branches.front().first.Kind);
   EXPECT_THAT(X.Branches.front().second.Chunks,
               ElementsAre(chunkKind(Chunk::K_Code)));
   // The missing terminating directive is marked as pp_not_keyword.
   EXPECT_EQ(tok::pp_not_keyword, X.End.Kind);
   EXPECT_EQ(0u, X.End.Tokens.size());
 }
 
+TEST(Preprocessor, StripComments) {
+  LangOptions Opts;
+  std::string Code = R"cpp(
+  // comment
+  int;
+)cpp";
+  TokenStream Raw = cook(lex(Code, Opts), Opts);
+  TokenStream Stripped = stripComments(Raw);
+  EXPECT_THAT(Raw.tokens(), ElementsAreArray({token("// comment", tok::comment),
+                                              token("int", tok::kw_int),
+                                              token(";", tok::semi)}));
+
+  EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+                                                   token(";", tok::semi)}));
+}
+
 } // namespace
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang