[clang-query] Add comment token handling

[steveire]

Summary

It is possible to pass a file of commands to clang-query using the
command line option -f or --preload. Make it possible to write comments
in such files.

[steveire]

Remove garbage

[aaron.ballman]

LGTM aside from a minor nit.

clang-query/QueryParser.cpp

42

This is just returning an empty StringRef, isn't it? Might as well return StringRef() through the default constructor for clarity.

[steveire]

clang-query/QueryParser.cpp

42

I copied the pattern from a few lines above, but there is it actually necessary as it is used in the code completion infrastructure. Comments don't participate in code completion so I changed it as described.

CMakeLists.txt

-add_subdirectory(clang-apply-replacements)
-add_subdirectory(clang-reorder-fields)
-add_subdirectory(modularize)
+# add_subdirectory(clang-apply-replacements)
+# add_subdirectory(clang-reorder-fields)
+# add_subdirectory(modularize)
 add_subdirectory(clang-tidy)
 add_subdirectory(clang-tidy-vs)
 
-add_subdirectory(change-namespace)
-add_subdirectory(clang-doc)
+# add_subdirectory(change-namespace)
+# add_subdirectory(clang-doc)
 add_subdirectory(clang-query)
-add_subdirectory(clang-move)
-add_subdirectory(clangd)
-add_subdirectory(include-fixer)
-add_subdirectory(pp-trace)
-add_subdirectory(tool-template)
+# add_subdirectory(clang-move)
+# # add_subdirectory(clangd)
+# add_subdirectory(include-fixer)
+# add_subdirectory(pp-trace)
+# add_subdirectory(tool-template)
 
 # Add the common testsuite after all the tools.
 if(CLANG_INCLUDE_TESTS)
-add_subdirectory(test)
+# add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
 
 option(CLANG_TOOLS_EXTRA_INCLUDE_DOCS "Generate build targets for the Clang Extra Tools docs."
   ${LLVM_INCLUDE_DOCS})
 if( CLANG_TOOLS_EXTRA_INCLUDE_DOCS )
   add_subdirectory(docs)
 endif()
 

clang-query/QueryParser.cpp

if (Begin == End)
       return StringRef(Begin, 0);
 
     if (!isWhitespace(*Begin))
       break;
 
     ++Begin;
   }
 
+  if (*Begin == '#') {
+    End = Begin;
+    return StringRef(Begin, 0);
+  }
+
   const char *WordBegin = Begin;
 
   while (true) {
     ++Begin;
 
     if (Begin == End || isWhitespace(*Begin))
       return StringRef(WordBegin, Begin - WordBegin);
   }
return new InvalidQuery("unexpected extra input: '" +
                             StringRef(Extra, End - Extra) + "'");
   return Q;
 }
 
 namespace {
 
 enum ParsedQueryKind {
   PQK_Invalid,
+  PQK_Comment,
   PQK_NoOp,
   PQK_Help,
   PQK_Let,
   PQK_Match,
   PQK_Set,
   PQK_Unlet,
   PQK_Quit
 };
QueryRef QueryParser::completeMatcherExpression() {
   }
   return QueryRef();
 }
 
 QueryRef QueryParser::doParse() {
   StringRef CommandStr;
   ParsedQueryKind QKind = LexOrCompleteWord<ParsedQueryKind>(this, CommandStr)
                               .Case("", PQK_NoOp)
+                              .Case("#", PQK_Comment, /*IsCompletion=*/false)
                               .Case("help", PQK_Help)
                               .Case("l", PQK_Let, /*IsCompletion=*/false)
                               .Case("let", PQK_Let)
                               .Case("m", PQK_Match, /*IsCompletion=*/false)
                               .Case("match", PQK_Match)
                               .Case("q", PQK_Quit,  /*IsCompletion=*/false)
                               .Case("quit", PQK_Quit)
                               .Case("set", PQK_Set)
                               .Case("unlet", PQK_Unlet)
                               .Default(PQK_Invalid);
 
   switch (QKind) {
+  case PQK_Comment:
   case PQK_NoOp:
     return new NoOpQuery;
 
   case PQK_Help:
     return endQuery(new HelpQuery);
 
   case PQK_Quit:
     return endQuery(new QuitQuery);

unittests/clang-query/QueryParserTest.cpp

TEST_F(QueryParserTest, LetUnlet) {
   EXPECT_EQ("expected variable name", cast<InvalidQuery>(Q)->ErrStr);
 
   Q = parse("unlet x bad_data");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("unexpected extra input: ' bad_data'",
             cast<InvalidQuery>(Q)->ErrStr);
 }
 
+TEST_F(QueryParserTest, Comment) {
+  QueryRef Q = parse("# let foo decl()");
+  ASSERT_TRUE(isa<NoOpQuery>(Q));
+
+  Q = parse("let foo decl() # creates a decl() matcher called foo");
+  ASSERT_TRUE(isa<LetQuery>(Q));
+
+  Q = parse("set bind-root false # reduce noise");
+  ASSERT_TRUE(isa<SetQuery<bool>>(Q));
+}
+
 TEST_F(QueryParserTest, Complete) {
   std::vector<llvm::LineEditor::Completion> Comps =
       QueryParser::complete("", 0, QS);
   ASSERT_EQ(6u, Comps.size());
   EXPECT_EQ("help ", Comps[0].TypedText);
   EXPECT_EQ("help", Comps[0].DisplayText);
   EXPECT_EQ("let ", Comps[1].TypedText);
   EXPECT_EQ("let", Comps[1].DisplayText);