Add completion to the query parser, and hook it up to clang-query.

[pcc]

• Add a test

[klimek]

clang-query/QueryParser.cpp

51

When reading this without the context of how it's used it's hard to know why this is templated, and what the template parameters are supposed to be.

64–71

I find this somewhat hard to read.
Things that might make it easier:

• pull out a constant for ~0u, or use one of the already existing constants
• s/SS/Switch/, or find an even more descriptive name what we're switching on here
• is there a particular reason you're using memcmp instead of using StringRefs and substr and operator==?

clang-query/QueryParser.h

23

While mostly self-explanatory, I still think the public interface might benefit from some comments...

53

I think some of the code later might be simpler if this was an offset from Begin. I might be missing something, though.

[pcc]

• Address review comments

clang-query/QueryParser.cpp

51

Added some comments (here and for lexOrCompleteWord).

64–71

pull out a constant for ~0u, or use one of the already existing constants

Done.

s/SS/Switch/, or find an even more descriptive name what we're switching on here

Done.

is there a particular reason you're using memcmp instead of using StringRefs and substr and operator==?

No particularly good reason. In fact I think the call to memcmp was a memory error. I've changed this as you suggested.

I also got rid of the SeenCases set; I replaced it with a new IsCompletion argument to Case which should make that logic more readable.

clang-query/QueryParser.h

23

Added.

53

The main reason I decided to make this a pointer was that Begin moves along the string as we parse, and if this were an offset we would need to keep it updated.

[klimek]

lg

[pcc]

Closed by commit rL200604 (authored by @pcc).

clang-query/QueryParser.h

 //===--- QueryParser.h - clang-query ----------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERY_PARSER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_QUERY_QUERY_PARSER_H
 
 #include "Query.h"
 
+#include <stddef.h>
+#include "llvm/LineEditor/LineEditor.h"
+
 namespace clang {
 namespace query {
 
-/// \brief Parse \p Line.
+class QuerySession;
+
+class QueryParser {
+ public:
+  static QueryRef parse(StringRef Line);
+
+  static std::vector<llvm::LineEditor::Completion> complete(StringRef Buffer,
+                                                            size_t Pos);
+
+ private:
+  QueryParser(StringRef Line)
+      : Begin(Line.data()), End(Line.data() + Line.size()), CompletionPos(0) {}
+
+  StringRef lexWord();
+
+  template <typename T> struct LexOrCompleteWord;
+  template <typename T> LexOrCompleteWord<T> lexOrCompleteWord(StringRef &Str);
+
+  QueryRef parseSetBool(bool QuerySession::*Var);
+  QueryRef parseSetOutputKind();
+
+  QueryRef endQuery(QueryRef Q);
+
+  /// \brief Parse [\p Begin,\p End).
   ///
   /// \return A reference to the parsed query object, which may be an
   /// \c InvalidQuery if a parse error occurs.
-QueryRef ParseQuery(StringRef Line);
+  QueryRef doParse();
+
+  const char *Begin;
+  const char *End;
+
+  const char *CompletionPos;
+  std::vector<llvm::LineEditor::Completion> Completions;
+};
 
 } // namespace query
 } // namespace clang
 
 #endif

clang-query/QueryParser.cpp

 #include "QueryParser.h"
 #include "Query.h"
 #include "QuerySession.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "clang/ASTMatchers/Dynamic/Parser.h"
 #include "clang/Basic/CharInfo.h"
 
+#include <set>
+
 using namespace llvm;
 using namespace clang::ast_matchers::dynamic;
 
 namespace clang {
 namespace query {
 
 // Lex any amount of whitespace followed by a "word" (any sequence of
 // non-whitespace characters) from the start of region [Begin,End).  If no word
 // is found before End, return StringRef().  Begin is adjusted to exclude the
 // lexed region.
-static StringRef LexWord(const char *&Begin, const char *End) {
+StringRef QueryParser::lexWord() {
   while (true) {
     if (Begin == End)
-      return StringRef();
+      return StringRef(Begin, 0);
 
     if (!isWhitespace(*Begin))
       break;
 
     ++Begin;
   }
 
   const char *WordBegin = Begin;
 
   while (true) {
     ++Begin;
 
     if (Begin == End || isWhitespace(*Begin))
       return StringRef(WordBegin, Begin - WordBegin);
   }
 }
 
-static QueryRef ParseSetBool(bool QuerySession::*Var, StringRef ValStr) {
-  unsigned Value = StringSwitch<unsigned>(ValStr)
+template <typename T> struct QueryParser::LexOrCompleteWord {
+  StringSwitch<T> SS;
+
+  QueryParser *P;
+  StringRef Str;
+  unsigned WordCompletionPos;
+  std::set<T> SeenCases;
+
+  LexOrCompleteWord(QueryParser *P, StringRef Str, unsigned WCP)
+      : SS(Str), P(P), Str(Str), WordCompletionPos(WCP) {}
+
+  template <unsigned N>
+  LexOrCompleteWord &Case(const char (&S)[N], const T &Value) {
+    if (WordCompletionPos == ~0u)
+      SS.Case(S, Value);
+    else if (N != 1 && SeenCases.insert(Value).second &&
+             std::memcmp(S, Str.data(), WordCompletionPos) == 0)
+      P->Completions.push_back(LineEditor::Completion(
+          std::string(S + WordCompletionPos, N - 1 - WordCompletionPos) + " ",
+          std::string(S, N - 1)));
+    return *this;
+  }
+
+  T Default(const T& Value) const {
+    return SS.Default(Value);
+  }
+};
+
+template <typename T>
+QueryParser::LexOrCompleteWord<T>
+QueryParser::lexOrCompleteWord(StringRef &Str) {
+  Str = lexWord();
+  unsigned WordCompletionPos = ~0u;
+  if (CompletionPos && CompletionPos <= Str.data() + Str.size()) {
+    if (CompletionPos < Str.data())
+      WordCompletionPos = 0;
+    else
+      WordCompletionPos = CompletionPos - Str.data();
+  }
+  return LexOrCompleteWord<T>(this, Str, WordCompletionPos);
+}
+
+QueryRef QueryParser::parseSetBool(bool QuerySession::*Var) {
+  StringRef ValStr;
+  unsigned Value = lexOrCompleteWord<unsigned>(ValStr)
                       .Case("false", 0)
                       .Case("true", 1)
                       .Default(~0u);
   if (Value == ~0u) {
     return new InvalidQuery("expected 'true' or 'false', got '" + ValStr + "'");
   }
   return new SetQuery<bool>(Var, Value);
 }
 
-static QueryRef ParseSetOutputKind(StringRef ValStr) {
-  unsigned OutKind = StringSwitch<unsigned>(ValStr)
+QueryRef QueryParser::parseSetOutputKind() {
+  StringRef ValStr;
+  unsigned OutKind = lexOrCompleteWord<unsigned>(ValStr)
                          .Case("diag", OK_Diag)
                          .Case("print", OK_Print)
                          .Case("dump", OK_Dump)
                          .Default(~0u);
   if (OutKind == ~0u) {
     return new InvalidQuery("expected 'diag', 'print' or 'dump', got '" +
                             ValStr + "'");
   }
   return new SetQuery<OutputKind>(&QuerySession::OutKind, OutputKind(OutKind));
 }
 
-static QueryRef EndQuery(const char *Begin, const char *End, QueryRef Q) {
+QueryRef QueryParser::endQuery(QueryRef Q) {
   const char *Extra = Begin;
-  if (!LexWord(Begin, End).empty())
+  if (!lexWord().empty())
     return new InvalidQuery("unexpected extra input: '" +
                             StringRef(Extra, End - Extra) + "'");
   return Q;
 }
 
 enum ParsedQueryKind {
   PQK_Invalid,
   PQK_NoOp,
   PQK_Help,
   PQK_Match,
   PQK_Set
 };
 
 enum ParsedQueryVariable {
   PQV_Invalid,
   PQV_Output,
   PQV_BindRoot
 };
 
-QueryRef ParseQuery(StringRef Line) {
-  const char *Begin = Line.data();
-  const char *End = Line.data() + Line.size();
-
-  StringRef CommandStr = LexWord(Begin, End);
-  ParsedQueryKind QKind = StringSwitch<ParsedQueryKind>(CommandStr)
+QueryRef QueryParser::doParse() {
+  StringRef CommandStr;
+  ParsedQueryKind QKind = lexOrCompleteWord<ParsedQueryKind>(CommandStr)
                               .Case("", PQK_NoOp)
                               .Case("help", PQK_Help)
-                              .Case("m", PQK_Match)
                               .Case("match", PQK_Match)
+                              .Case("m", PQK_Match)
                               .Case("set", PQK_Set)
                               .Default(PQK_Invalid);
 
   switch (QKind) {
   case PQK_NoOp:
     return new NoOpQuery;
 
   case PQK_Help:
-    return EndQuery(Begin, End, new HelpQuery);
+    return endQuery(new HelpQuery);
 
   case PQK_Match: {
+    if (CompletionPos) {
+      std::vector<MatcherCompletion> Comps = Parser::completeExpression(
+          StringRef(Begin, End - Begin), CompletionPos - Begin);
+      for (std::vector<MatcherCompletion>::iterator I = Comps.begin(),
+                                                    E = Comps.end();
+           I != E; ++I) {
+        Completions.push_back(
+            LineEditor::Completion(I->TypedText, I->MatcherDecl));
+      }
+      return QueryRef();
+    } else {
       Diagnostics Diag;
       Optional<DynTypedMatcher> Matcher =
           Parser::parseMatcherExpression(StringRef(Begin, End - Begin), &Diag);
       if (!Matcher) {
         std::string ErrStr;
         llvm::raw_string_ostream OS(ErrStr);
         Diag.printToStreamFull(OS);
         return new InvalidQuery(OS.str());
       }
       return new MatchQuery(*Matcher);
     }
+  }
 
   case PQK_Set: {
-    StringRef VarStr = LexWord(Begin, End);
-    if (VarStr.empty())
-      return new InvalidQuery("expected variable name");
-
-    ParsedQueryVariable Var = StringSwitch<ParsedQueryVariable>(VarStr)
+    StringRef VarStr;
+    ParsedQueryVariable Var = lexOrCompleteWord<ParsedQueryVariable>(VarStr)
                                   .Case("output", PQV_Output)
                                   .Case("bind-root", PQV_BindRoot)
                                   .Default(PQV_Invalid);
+    if (VarStr.empty())
+      return new InvalidQuery("expected variable name");
     if (Var == PQV_Invalid)
       return new InvalidQuery("unknown variable: '" + VarStr + "'");
 
-    StringRef ValStr = LexWord(Begin, End);
-    if (ValStr.empty())
-      return new InvalidQuery("expected variable value");
-
     QueryRef Q;
     switch (Var) {
     case PQV_Output:
-      Q = ParseSetOutputKind(ValStr);
+      Q = parseSetOutputKind();
       break;
     case PQV_BindRoot:
-      Q = ParseSetBool(&QuerySession::BindRoot, ValStr);
+      Q = parseSetBool(&QuerySession::BindRoot);
       break;
     case PQV_Invalid:
       llvm_unreachable("Invalid query kind");
     }
 
-    return EndQuery(Begin, End, Q);
+    return endQuery(Q);
   }
 
   case PQK_Invalid:
     return new InvalidQuery("unknown command: " + CommandStr);
   }
 
   llvm_unreachable("Invalid query kind");
 }
 
+QueryRef QueryParser::parse(StringRef Line) {
+  return QueryParser(Line).doParse();
+}
+
+std::vector<LineEditor::Completion> QueryParser::complete(StringRef Line,
+                                                          size_t Pos) {
+  QueryParser P(Line);
+  P.CompletionPos = Line.data() + Pos;
+
+  P.doParse();
+  return P.Completions;
+}
+
 } // namespace query
 } // namespace clang

clang-query/tool/ClangQuery.cpp

if (Tool.buildASTs(ASTs) != 0)
     return 1;
 
   QuerySession QS(ASTs);
 
   if (!Commands.empty()) {
     for (cl::list<std::string>::iterator I = Commands.begin(),
                                          E = Commands.end();
          I != E; ++I) {
-      QueryRef Q = ParseQuery(I->c_str());
+      QueryRef Q = QueryParser::parse(I->c_str());
       if (!Q->run(llvm::outs(), QS))
         return 1;
     }
   } else if (!CommandFiles.empty()) {
     for (cl::list<std::string>::iterator I = CommandFiles.begin(),
                                          E = CommandFiles.end();
          I != E; ++I) {
       std::ifstream Input(I->c_str());
       if (!Input.is_open()) {
         llvm::errs() << argv[0] << ": cannot open " << *I << "\n";
         return 1;
       }
       while (Input.good()) {
         std::string Line;
         std::getline(Input, Line);
 
-        QueryRef Q = ParseQuery(Line.c_str());
+        QueryRef Q = QueryParser::parse(Line.c_str());
         if (!Q->run(llvm::outs(), QS))
           return 1;
       }
     }
   } else {
     LineEditor LE("clang-query");
+    LE.setListCompleter(QueryParser::complete);
     while (llvm::Optional<std::string> Line = LE.readLine()) {
-      QueryRef Q = ParseQuery(*Line);
+      QueryRef Q = QueryParser::parse(*Line);
       Q->run(llvm::outs(), QS);
     }
   }
 
   llvm::DeleteContainerPointers(ASTs);
 
   return 0;
 }

unittests/clang-query/QueryParserTest.cpp

 //===---- QueryParserTest.cpp - clang-query test --------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "QueryParser.h"
 #include "Query.h"
 #include "QuerySession.h"
 #include "gtest/gtest.h"
+#include "llvm/LineEditor/LineEditor.h"
 
 using namespace clang;
 using namespace clang::query;
 
 TEST(QueryParser, NoOp) {
-  QueryRef Q = ParseQuery("");
+  QueryRef Q = QueryParser::parse("");
   EXPECT_TRUE(isa<NoOpQuery>(Q));
 
-  Q = ParseQuery("\n");
+  Q = QueryParser::parse("\n");
   EXPECT_TRUE(isa<NoOpQuery>(Q));
 }
 
 TEST(QueryParser, Invalid) {
-  QueryRef Q = ParseQuery("foo");
+  QueryRef Q = QueryParser::parse("foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("unknown command: foo", cast<InvalidQuery>(Q)->ErrStr);
 }
 
 TEST(QueryParser, Help) {
-  QueryRef Q = ParseQuery("help");
+  QueryRef Q = QueryParser::parse("help");
   ASSERT_TRUE(isa<HelpQuery>(Q));
 
-  Q = ParseQuery("help me");
+  Q = QueryParser::parse("help me");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("unexpected extra input: ' me'", cast<InvalidQuery>(Q)->ErrStr);
 }
 
 TEST(QueryParser, Set) {
-  QueryRef Q = ParseQuery("set");
+  QueryRef Q = QueryParser::parse("set");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("expected variable name", cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set foo bar");
+  Q = QueryParser::parse("set foo bar");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("unknown variable: 'foo'", cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set output");
+  Q = QueryParser::parse("set output");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
-  EXPECT_EQ("expected variable value", cast<InvalidQuery>(Q)->ErrStr);
+  EXPECT_EQ("expected 'diag', 'print' or 'dump', got ''",
+            cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set bind-root true foo");
+  Q = QueryParser::parse("set bind-root true foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("unexpected extra input: ' foo'", cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set output foo");
+  Q = QueryParser::parse("set output foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("expected 'diag', 'print' or 'dump', got 'foo'",
             cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set output dump");
+  Q = QueryParser::parse("set output dump");
   ASSERT_TRUE(isa<SetQuery<OutputKind> >(Q));
   EXPECT_EQ(&QuerySession::OutKind, cast<SetQuery<OutputKind> >(Q)->Var);
   EXPECT_EQ(OK_Dump, cast<SetQuery<OutputKind> >(Q)->Value);
 
-  Q = ParseQuery("set bind-root foo");
+  Q = QueryParser::parse("set bind-root foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("expected 'true' or 'false', got 'foo'",
             cast<InvalidQuery>(Q)->ErrStr);
 
-  Q = ParseQuery("set bind-root true");
+  Q = QueryParser::parse("set bind-root true");
   ASSERT_TRUE(isa<SetQuery<bool> >(Q));
   EXPECT_EQ(&QuerySession::BindRoot, cast<SetQuery<bool> >(Q)->Var);
   EXPECT_EQ(true, cast<SetQuery<bool> >(Q)->Value);
 }
 
 TEST(QueryParser, Match) {
-  QueryRef Q = ParseQuery("match decl()");
+  QueryRef Q = QueryParser::parse("match decl()");
   ASSERT_TRUE(isa<MatchQuery>(Q));
   EXPECT_TRUE(cast<MatchQuery>(Q)->Matcher.canConvertTo<Decl>());
 
-  Q = ParseQuery("m stmt()");
+  Q = QueryParser::parse("m stmt()");
   ASSERT_TRUE(isa<MatchQuery>(Q));
   EXPECT_TRUE(cast<MatchQuery>(Q)->Matcher.canConvertTo<Stmt>());
 }
+
+TEST(QueryParser, Complete) {
+  std::vector<llvm::LineEditor::Completion> Comps =
+      QueryParser::complete("", 0);
+  ASSERT_EQ(3u, Comps.size());
+  EXPECT_EQ("help ", Comps[0].TypedText);
+  EXPECT_EQ("help", Comps[0].DisplayText);
+  EXPECT_EQ("match ", Comps[1].TypedText);
+  EXPECT_EQ("match", Comps[1].DisplayText);
+  EXPECT_EQ("set ", Comps[2].TypedText);
+  EXPECT_EQ("set", Comps[2].DisplayText);
+
+  Comps = QueryParser::complete("set o", 5);
+  ASSERT_EQ(1u, Comps.size());
+  EXPECT_EQ("utput ", Comps[0].TypedText);
+  EXPECT_EQ("output", Comps[0].DisplayText);
+
+  Comps = QueryParser::complete("match while", 11);
+  ASSERT_EQ(1u, Comps.size());
+  EXPECT_EQ("Stmt(", Comps[0].TypedText);
+  EXPECT_EQ("Matcher<Stmt> whileStmt(Matcher<WhileStmt>...)",
+            Comps[0].DisplayText);
+}