Introduce Parser::completeExpression.

[pcc]

Summary

This function returns a list of completions for a given expression and
completion position.

[sbenza]

lib/ASTMatchers/Dynamic/Parser.cpp

294 ↗

(On Diff #5752)

Can you add a scoped object that will pop_back automatically?

388 ↗

(On Diff #5752)

Is there a good use case for not aborting the parse if lookupMatcherCtor() returns null?
(instead of pushing it into the stack and then checking if there is a null in the stack)

[pcc]

• Use a scoped object to manage the context stack

lib/ASTMatchers/Dynamic/Parser.cpp

294 ↗

(On Diff #5752)

Done.

388 ↗

(On Diff #5752)

Not aborting gives us better (i.e. more) diagnostics. If we abort early, no diagnostics appear for anything after the invalid matcher name. (This also causes some unit tests under ParserTest.Errors to fail.)

[pcc]

Closed by commit rL200497 (authored by @pcc).

cfe/trunk/include/clang/ASTMatchers/Dynamic/Parser.h

public:
   /// \brief Parse an expression.
   ///
   /// Parses any expression supported by this parser. In general, the
   /// \c parseMatcherExpression function is a better approach to get a matcher
   /// object.
   static bool parseExpression(StringRef Code, Sema *S,
                               VariantValue *Value, Diagnostics *Error);
 
+  /// \brief Complete an expression at the given offset.
+  ///
+  /// \return The list of completions, which may be empty if there are no
+  /// available completions or if an error occurred.
+  static std::vector<MatcherCompletion>
+  completeExpression(StringRef Code, unsigned CompletionOffset);
+
 private:
   class CodeTokenizer;
+  struct ScopedContextEntry;
   struct TokenInfo;
 
   Parser(CodeTokenizer *Tokenizer, Sema *S,
          Diagnostics *Error);
 
   bool parseExpressionImpl(VariantValue *Value);
   bool parseMatcherExpressionImpl(VariantValue *Value);
 
+  void addCompletion(const TokenInfo &CompToken, StringRef TypedText,
+                     StringRef Decl);
+  void addExpressionCompletions();
+
   CodeTokenizer *const Tokenizer;
   Sema *const S;
   Diagnostics *const Error;
+
+  typedef std::vector<std::pair<MatcherCtor, unsigned> > ContextStackTy;
+  ContextStackTy ContextStack;
+  std::vector<MatcherCompletion> Completions;
 };
 
 }  // namespace dynamic
 }  // namespace ast_matchers
 }  // namespace clang
 
 #endif  // LLVM_CLANG_AST_MATCHERS_DYNAMIC_PARSER_H

cfe/trunk/lib/ASTMatchers/Dynamic/Parser.cpp

 namespace clang {
 namespace ast_matchers {
 namespace dynamic {
 
 /// \brief Simple structure to hold information for one token from the parser.
 struct Parser::TokenInfo {
   /// \brief Different possible tokens.
   enum TokenKind {
-    TK_Eof = 0,
-    TK_OpenParen = 1,
-    TK_CloseParen = 2,
-    TK_Comma = 3,
-    TK_Period = 4,
-    TK_Literal = 5,
-    TK_Ident = 6,
-    TK_InvalidChar = 7,
-    TK_Error = 8
+    TK_Eof,
+    TK_OpenParen,
+    TK_CloseParen,
+    TK_Comma,
+    TK_Period,
+    TK_Literal,
+    TK_Ident,
+    TK_InvalidChar,
+    TK_Error,
+    TK_CodeCompletion
   };
 
   /// \brief Some known identifiers.
   static const char* const ID_Bind;
 
   TokenInfo() : Text(), Kind(TK_Eof), Range(), Value() {}
 
   StringRef Text;
   TokenKind Kind;
   SourceRange Range;
   VariantValue Value;
 };
 
 const char* const Parser::TokenInfo::ID_Bind = "bind";
 
 /// \brief Simple tokenizer for the parser.
 class Parser::CodeTokenizer {
 public:
   explicit CodeTokenizer(StringRef MatcherCode, Diagnostics *Error)
-      : Code(MatcherCode), StartOfLine(MatcherCode), Line(1), Error(Error) {
+      : Code(MatcherCode), StartOfLine(MatcherCode), Line(1), Error(Error),
+        CodeCompletionLocation(0) {
+    NextToken = getNextToken();
+  }
+
+  CodeTokenizer(StringRef MatcherCode, Diagnostics *Error,
+                unsigned CodeCompletionOffset)
+      : Code(MatcherCode), StartOfLine(MatcherCode), Line(1), Error(Error),
+        CodeCompletionLocation(MatcherCode.data() + CodeCompletionOffset) {
     NextToken = getNextToken();
   }
 
   /// \brief Returns but doesn't consume the next token.
   const TokenInfo &peekNextToken() const { return NextToken; }
 
   /// \brief Consumes and returns the next token.
   TokenInfo consumeNextToken() {
     TokenInfo ThisToken = NextToken;
     NextToken = getNextToken();
     return ThisToken;
   }
 
   TokenInfo::TokenKind nextTokenKind() const { return NextToken.Kind; }
 
 private:
   TokenInfo getNextToken() {
     consumeWhitespace();
     TokenInfo Result;
     Result.Range.Start = currentLocation();
 
+    if (CodeCompletionLocation && CodeCompletionLocation <= Code.data()) {
+      Result.Kind = TokenInfo::TK_CodeCompletion;
+      Result.Text = StringRef(CodeCompletionLocation, 0);
+      CodeCompletionLocation = 0;
+      return Result;
+    }
+
     if (Code.empty()) {
       Result.Kind = TokenInfo::TK_Eof;
       Result.Text = "";
       return Result;
     }
 
     switch (Code[0]) {
     case ',':
case '5': case '6': case '7': case '8': case '9':
       // Parse an unsigned literal.
       consumeUnsignedLiteral(&Result);
       break;
 
     default:
       if (isAlphanumeric(Code[0])) {
         // Parse an identifier
         size_t TokenLength = 1;
-        while (TokenLength < Code.size() && isAlphanumeric(Code[TokenLength]))
+        while (1) {
+          // A code completion location in/immediately after an identifier will
+          // cause the portion of the identifier before the code completion
+          // location to become a code completion token.
+          if (CodeCompletionLocation == Code.data() + TokenLength) {
+            CodeCompletionLocation = 0;
+            Result.Kind = TokenInfo::TK_CodeCompletion;
+            Result.Text = Code.substr(0, TokenLength);
+            Code = Code.drop_front(TokenLength);
+            return Result;
+          }
+          if (TokenLength == Code.size() || !isAlphanumeric(Code[TokenLength]))
+            break;
           ++TokenLength;
+        }
         Result.Kind = TokenInfo::TK_Ident;
         Result.Text = Code.substr(0, TokenLength);
         Code = Code.drop_front(TokenLength);
       } else {
         Result.Kind = TokenInfo::TK_InvalidChar;
         Result.Text = Code.substr(0, 1);
         Code = Code.drop_front(1);
       }
SourceLocation currentLocation() {
     return Location;
   }
 
   StringRef Code;
   StringRef StartOfLine;
   unsigned Line;
   Diagnostics *Error;
   TokenInfo NextToken;
+  const char *CodeCompletionLocation;
 };
 
 Parser::Sema::~Sema() {}
 
+struct Parser::ScopedContextEntry {
+  Parser *P;
+
+  ScopedContextEntry(Parser *P, MatcherCtor C) : P(P) {
+    P->ContextStack.push_back(std::make_pair(C, 0u));
+  }
+
+  ~ScopedContextEntry() {
+    P->ContextStack.pop_back();
+  }
+
+  void nextArg() {
+    ++P->ContextStack.back().second;
+  }
+};
+
 /// \brief Parse and validate a matcher expression.
 /// \return \c true on success, in which case \c Value has the matcher parsed.
 ///   If the input is malformed, or some argument has an error, it
 ///   returns \c false.
 bool Parser::parseMatcherExpressionImpl(VariantValue *Value) {
   const TokenInfo NameToken = Tokenizer->consumeNextToken();
   assert(NameToken.Kind == TokenInfo::TK_Ident);
   const TokenInfo OpenToken = Tokenizer->consumeNextToken();
   if (OpenToken.Kind != TokenInfo::TK_OpenParen) {
     Error->addError(OpenToken.Range, Error->ET_ParserNoOpenParen)
         << OpenToken.Text;
     return false;
   }
 
   llvm::Optional<MatcherCtor> Ctor =
       S->lookupMatcherCtor(NameToken.Text, NameToken.Range, Error);
-
   std::vector<ParserValue> Args;
   TokenInfo EndToken;
+
+  {
+    ScopedContextEntry SCE(this, Ctor ? *Ctor : 0);
+
     while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
       if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
         // End of args.
         EndToken = Tokenizer->consumeNextToken();
         break;
       }
       if (Args.size() > 0) {
         // We must find a , token to continue.
         const TokenInfo CommaToken = Tokenizer->consumeNextToken();
         if (CommaToken.Kind != TokenInfo::TK_Comma) {
           Error->addError(CommaToken.Range, Error->ET_ParserNoComma)
               << CommaToken.Text;
           return false;
         }
       }
 
       Diagnostics::Context Ctx(Diagnostics::Context::MatcherArg, Error,
-                             NameToken.Text, NameToken.Range, Args.size() + 1);
+                               NameToken.Text, NameToken.Range,
+                               Args.size() + 1);
       ParserValue ArgValue;
       ArgValue.Text = Tokenizer->peekNextToken().Text;
       ArgValue.Range = Tokenizer->peekNextToken().Range;
-    if (!parseExpressionImpl(&ArgValue.Value)) return false;
+      if (!parseExpressionImpl(&ArgValue.Value)) {
+        return false;
+      }
 
       Args.push_back(ArgValue);
+      SCE.nextArg();
+    }
   }
 
   if (EndToken.Kind == TokenInfo::TK_Eof) {
     Error->addError(OpenToken.Range, Error->ET_ParserNoCloseParen);
     return false;
   }
 
   std::string BindID;
   if (Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period) {
     // Parse .bind("foo")
     Tokenizer->consumeNextToken();  // consume the period.
     const TokenInfo BindToken = Tokenizer->consumeNextToken();
+    if (BindToken.Kind == TokenInfo::TK_CodeCompletion) {
+      addCompletion(BindToken, "bind(\"", "bind");
+      return false;
+    }
+
     const TokenInfo OpenToken = Tokenizer->consumeNextToken();
     const TokenInfo IDToken = Tokenizer->consumeNextToken();
     const TokenInfo CloseToken = Tokenizer->consumeNextToken();
 
     // TODO: We could use different error codes for each/some to be more
     //       explicit about the syntax error.
     if (BindToken.Kind != TokenInfo::TK_Ident ||
         BindToken.Text != TokenInfo::ID_Bind) {
bool Parser::parseMatcherExpressionImpl(VariantValue *Value) {
   VariantMatcher Result = S->actOnMatcherExpression(
       *Ctor, MatcherRange, BindID, Args, Error);
   if (Result.isNull()) return false;
 
   *Value = Result;
   return true;
 }
 
+// If the prefix of this completion matches the completion token, add it to
+// Completions minus the prefix.
+void Parser::addCompletion(const TokenInfo &CompToken, StringRef TypedText,
+                           StringRef Decl) {
+  if (TypedText.size() >= CompToken.Text.size() &&
+      TypedText.substr(0, CompToken.Text.size()) == CompToken.Text) {
+    Completions.push_back(
+        MatcherCompletion(TypedText.substr(CompToken.Text.size()), Decl));
+  }
+}
+
+void Parser::addExpressionCompletions() {
+  const TokenInfo CompToken = Tokenizer->consumeNextToken();
+  assert(CompToken.Kind == TokenInfo::TK_CodeCompletion);
+
+  // We cannot complete code if there is an invalid element on the context
+  // stack.
+  for (ContextStackTy::iterator I = ContextStack.begin(),
+                                E = ContextStack.end();
+       I != E; ++I) {
+    if (!I->first)
+      return;
+  }
+
+  std::vector<MatcherCompletion> RegCompletions =
+      Registry::getCompletions(ContextStack);
+  for (std::vector<MatcherCompletion>::iterator I = RegCompletions.begin(),
+                                                E = RegCompletions.end();
+       I != E; ++I) {
+    addCompletion(CompToken, I->TypedText, I->MatcherDecl);
+  }
+}
+
 /// \brief Parse an <Expresssion>
 bool Parser::parseExpressionImpl(VariantValue *Value) {
   switch (Tokenizer->nextTokenKind()) {
   case TokenInfo::TK_Literal:
     *Value = Tokenizer->consumeNextToken().Value;
     return true;
 
   case TokenInfo::TK_Ident:
     return parseMatcherExpressionImpl(Value);
 
+  case TokenInfo::TK_CodeCompletion:
+    addExpressionCompletions();
+    return false;
+
   case TokenInfo::TK_Eof:
     Error->addError(Tokenizer->consumeNextToken().Range,
                     Error->ET_ParserNoCode);
     return false;
 
   case TokenInfo::TK_Error:
     // This error was already reported by the tokenizer.
     return false;
bool Parser::parseExpression(StringRef Code, Sema *S,
   if (Tokenizer.peekNextToken().Kind != TokenInfo::TK_Eof) {
     Error->addError(Tokenizer.peekNextToken().Range,
                     Error->ET_ParserTrailingCode);
     return false;
   }
   return true;
 }
 
+std::vector<MatcherCompletion>
+Parser::completeExpression(StringRef Code, unsigned CompletionOffset) {
+  Diagnostics Error;
+  CodeTokenizer Tokenizer(Code, &Error, CompletionOffset);
+  RegistrySema S;
+  Parser P(&Tokenizer, &S, &Error);
+  VariantValue Dummy;
+  P.parseExpressionImpl(&Dummy);
+
+  return P.Completions;
+}
+
 llvm::Optional<DynTypedMatcher>
 Parser::parseMatcherExpression(StringRef Code, Diagnostics *Error) {
   RegistrySema S;
   return parseMatcherExpression(Code, &S, Error);
 }
 
 llvm::Optional<DynTypedMatcher>
 Parser::parseMatcherExpression(StringRef Code, Parser::Sema *S,

cfe/trunk/unittests/ASTMatchers/Dynamic/ParserTest.cpp

TEST(ParserTest, OverloadErrors) {
   EXPECT_EQ("1:1: Error building matcher callee.\n"
             "1:8: Candidate 1: Incorrect type for arg 1. "
             "(Expected = Matcher<Stmt>) != (Actual = String)\n"
             "1:8: Candidate 2: Incorrect type for arg 1. "
             "(Expected = Matcher<Decl>) != (Actual = String)",
             ParseWithError("callee(\"A\")"));
 }
 
+TEST(ParserTest, Completion) {
+  std::vector<MatcherCompletion> Comps =
+      Parser::completeExpression("while", 5);
+  ASSERT_EQ(1u, Comps.size());
+  EXPECT_EQ("Stmt(", Comps[0].TypedText);
+  EXPECT_EQ("Matcher<Stmt> whileStmt(Matcher<WhileStmt>...)",
+            Comps[0].MatcherDecl);
+
+  Comps = Parser::completeExpression("whileStmt().", 12);
+  ASSERT_EQ(1u, Comps.size());
+  EXPECT_EQ("bind(\"", Comps[0].TypedText);
+  EXPECT_EQ("bind", Comps[0].MatcherDecl);
+}
+
 }  // end anonymous namespace
 }  // end namespace dynamic
 }  // end namespace ast_matchers
 }  // end namespace clang